From ab9e8d06ee719553e167042fd63e094c1a6c97ee Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 7 Feb 2025 09:50:05 -0800 Subject: [PATCH 001/226] Add a delayed logging handler that delays anything logged before logging is actually configured. --- borgmatic/commands/arguments.py | 4 +- borgmatic/commands/borgmatic.py | 2 + borgmatic/logger.py | 59 ++++++++++++++++++++++- tests/unit/test_logger.py | 83 +++++++++++++++++++++++++++++++++ 4 files changed, 145 insertions(+), 3 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 1252270b..e95ec5f4 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -349,12 +349,12 @@ def make_parsers(): global_group.add_argument( '--log-file-format', type=str, - help='Log format string used for log messages written to the log file', + help='Python format string used for log messages written to the log file', ) global_group.add_argument( '--log-json', action='store_true', - help='Write log messages and console output as one JSON object per log line instead of formatted text', + help='Write Borg log messages and console output as one JSON object per log line instead of formatted text', ) global_group.add_argument( '--override', diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 76f16a43..8f677191 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -43,6 +43,7 @@ from borgmatic.logger import ( DISABLED, Log_prefix, add_custom_log_levels, + configure_delayed_logging, configure_logging, should_do_markup, ) @@ -880,6 +881,7 @@ def exit_with_help_link(): # pragma: no cover def main(extra_summary_logs=[]): # pragma: no cover configure_signals() + configure_delayed_logging() try: arguments = parse_arguments(*sys.argv[1:]) diff --git a/borgmatic/logger.py b/borgmatic/logger.py index 4fb03a58..1fc0da52 100644 --- a/borgmatic/logger.py +++ b/borgmatic/logger.py @@ -263,6 +263,62 @@ class Log_prefix: set_log_prefix(self.original_prefix) +class Delayed_logging_handler(logging.handlers.BufferingHandler): + ''' + A logging handler that buffers logs and doesn't flush them until explicitly flushed (after + target handlers are actually set). It's useful for holding onto messages logged before logging + is configured, ensuring those records eventually make their way to the relevant logging + handlers. + ''' + + def __init__(self): + super(Delayed_logging_handler, self).__init__(capacity=0) + + self.targets = None + + def shouldFlush(self, record): + return self.targets is not None + + def flush(self): + self.acquire() + + try: + if not self.targets: + return + + for record in self.buffer: + for target in self.targets: + target.handle(record) + + self.buffer.clear() + finally: + self.release() + + +def configure_delayed_logging(): # pragma: no cover + ''' + Configure a delayed logging handler to buffer anything that gets logged until we're ready to + deal with it. + ''' + logging.basicConfig( + level=logging.DEBUG, + handlers=[Delayed_logging_handler()], + ) + + +def flush_delayed_logging(target_handlers): + ''' + Flush any previously buffered logs to our "real" logging handlers. + ''' + root_logger = logging.getLogger() + + if root_logger.handlers and isinstance(root_logger.handlers[0], Delayed_logging_handler): + delayed_handler = root_logger.handlers[0] + delayed_handler.targets = target_handlers + delayed_handler.flush() + root_logger.removeHandler(delayed_handler) + + def configure_logging( console_log_level, syslog_log_level=None, @@ -310,7 +366,6 @@ def configure_logging( console_handler.setFormatter(Log_prefix_formatter()) console_handler.setLevel(console_log_level) - handlers = [console_handler] if syslog_log_level != logging.DISABLED: @@ -343,6 +398,8 @@ def configure_logging( file_handler.setLevel(log_file_log_level) handlers.append(file_handler) + flush_delayed_logging(handlers) + logging.basicConfig( level=min(handler.level for handler in handlers), handlers=handlers, diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index 5f0fae12..a1222862 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -348,6 +348,78 @@ def test_log_prefix_sets_prefix_and_then_restores_original_prefix_after(): pass +def test_delayed_logging_handler_should_flush_without_targets_returns_false(): + handler = module.Delayed_logging_handler() + + assert handler.shouldFlush(flexmock()) is False + + +def test_delayed_logging_handler_should_flush_with_targets_returns_true(): + handler = module.Delayed_logging_handler() + handler.targets = [flexmock()] + + assert handler.shouldFlush(flexmock()) is True + + +def test_delayed_logging_handler_flush_without_targets_does_not_raise(): + handler = module.Delayed_logging_handler() + flexmock(handler).should_receive('acquire') + flexmock(handler).should_receive('release') + + handler.flush() + + +def test_delayed_logging_handler_flush_with_empty_buffer_does_not_raise(): + handler = module.Delayed_logging_handler() + flexmock(handler).should_receive('acquire') + flexmock(handler).should_receive('release') + handler.targets = [flexmock()] + + handler.flush() + + +def test_delayed_logging_handler_flush_forwards_each_record_to_each_target(): + handler = module.Delayed_logging_handler() + flexmock(handler).should_receive('acquire') + flexmock(handler).should_receive('release') + handler.targets = [flexmock(), flexmock()] + handler.buffer = [flexmock(), flexmock()] + handler.targets[0].should_receive('handle').with_args(handler.buffer[0]).once() + handler.targets[1].should_receive('handle').with_args(handler.buffer[0]).once() + handler.targets[0].should_receive('handle').with_args(handler.buffer[1]).once() + handler.targets[1].should_receive('handle').with_args(handler.buffer[1]).once() + + handler.flush() + + assert handler.buffer == [] + + +def test_flush_delayed_logging_without_handlers_does_not_raise(): + root_logger = flexmock(handlers=[]) + root_logger.should_receive('removeHandler') + flexmock(module.logging).should_receive('getLogger').and_return(root_logger) + + module.flush_delayed_logging([flexmock()]) + + +def test_flush_delayed_logging_without_delayed_logging_handler_does_not_raise(): + root_logger = flexmock(handlers=[flexmock()]) + root_logger.should_receive('removeHandler') + flexmock(module.logging).should_receive('getLogger').and_return(root_logger) + + module.flush_delayed_logging([flexmock()]) + + +def test_flush_delayed_logging_flushes_delayed_logging_handler(): + delayed_logging_handler = module.Delayed_logging_handler() + root_logger = flexmock(handlers=[delayed_logging_handler]) + flexmock(module.logging).should_receive('getLogger').and_return(root_logger) + flexmock(delayed_logging_handler).should_receive('flush').once() + root_logger.should_receive('removeHandler') + + module.flush_delayed_logging([flexmock()]) + + def test_configure_logging_with_syslog_log_level_probes_for_log_socket_on_linux(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.ANSWER @@ -357,6 +429,7 @@ def test_configure_logging_with_syslog_log_level_probes_for_log_socket_on_linux( multi_stream_handler.should_receive('setFormatter').with_args(fake_formatter).once() flexmock(module).should_receive('Multi_stream_handler').and_return(multi_stream_handler) flexmock(module).should_receive('interactive_console').and_return(False) + flexmock(module).should_receive('flush_delayed_logging') flexmock(module.logging).should_receive('basicConfig').with_args( level=logging.DEBUG, handlers=list ) @@ -378,6 +451,7 @@ def test_configure_logging_with_syslog_log_level_probes_for_log_socket_on_macos( multi_stream_handler.should_receive('setFormatter').with_args(fake_formatter).once() flexmock(module).should_receive('Multi_stream_handler').and_return(multi_stream_handler) flexmock(module).should_receive('interactive_console').and_return(False) + flexmock(module).should_receive('flush_delayed_logging') flexmock(module.logging).should_receive('basicConfig').with_args( level=logging.DEBUG, handlers=list ) @@ -400,6 +474,7 @@ def test_configure_logging_with_syslog_log_level_probes_for_log_socket_on_freebs multi_stream_handler.should_receive('setFormatter').with_args(fake_formatter).once() flexmock(module).should_receive('Multi_stream_handler').and_return(multi_stream_handler) flexmock(module).should_receive('interactive_console').and_return(False) + flexmock(module).should_receive('flush_delayed_logging') flexmock(module.logging).should_receive('basicConfig').with_args( level=logging.DEBUG, handlers=list ) @@ -422,6 +497,7 @@ def test_configure_logging_without_syslog_log_level_skips_syslog(): multi_stream_handler = flexmock(setLevel=lambda level: None, level=logging.INFO) multi_stream_handler.should_receive('setFormatter').with_args(fake_formatter).once() flexmock(module).should_receive('Multi_stream_handler').and_return(multi_stream_handler) + flexmock(module).should_receive('flush_delayed_logging') flexmock(module.logging).should_receive('basicConfig').with_args( level=logging.INFO, handlers=list ) @@ -439,6 +515,7 @@ def test_configure_logging_skips_syslog_if_not_found(): multi_stream_handler = flexmock(setLevel=lambda level: None, level=logging.INFO) multi_stream_handler.should_receive('setFormatter').with_args(fake_formatter).once() flexmock(module).should_receive('Multi_stream_handler').and_return(multi_stream_handler) + flexmock(module).should_receive('flush_delayed_logging') flexmock(module.logging).should_receive('basicConfig').with_args( level=logging.INFO, handlers=list ) @@ -457,6 +534,7 @@ def test_configure_logging_skips_log_file_if_log_file_logging_is_disabled(): multi_stream_handler.should_receive('setFormatter').with_args(fake_formatter).once() flexmock(module).should_receive('Multi_stream_handler').and_return(multi_stream_handler) + flexmock(module).should_receive('flush_delayed_logging') flexmock(module.logging).should_receive('basicConfig').with_args( level=logging.INFO, handlers=list ) @@ -478,6 +556,7 @@ def test_configure_logging_to_log_file_instead_of_syslog(): multi_stream_handler.should_receive('setFormatter').with_args(fake_formatter).once() flexmock(module).should_receive('Multi_stream_handler').and_return(multi_stream_handler) + flexmock(module).should_receive('flush_delayed_logging') flexmock(module.logging).should_receive('basicConfig').with_args( level=logging.DEBUG, handlers=list ) @@ -505,6 +584,7 @@ def test_configure_logging_to_both_log_file_and_syslog(): multi_stream_handler.should_receive('setFormatter').with_args(fake_formatter).once() flexmock(module).should_receive('Multi_stream_handler').and_return(multi_stream_handler) + flexmock(module).should_receive('flush_delayed_logging') flexmock(module.logging).should_receive('basicConfig').with_args( level=logging.DEBUG, handlers=list ) @@ -539,6 +619,7 @@ def test_configure_logging_to_log_file_formats_with_custom_log_format(): flexmock(module).should_receive('Multi_stream_handler').and_return(multi_stream_handler) flexmock(module).should_receive('interactive_console').and_return(False) + flexmock(module).should_receive('flush_delayed_logging') flexmock(module.logging).should_receive('basicConfig').with_args( level=logging.DEBUG, handlers=list ) @@ -566,6 +647,7 @@ def test_configure_logging_skips_log_file_if_argument_is_none(): multi_stream_handler.should_receive('setFormatter').with_args(fake_formatter).once() flexmock(module).should_receive('Multi_stream_handler').and_return(multi_stream_handler) + flexmock(module).should_receive('flush_delayed_logging') flexmock(module.logging).should_receive('basicConfig').with_args( level=logging.INFO, handlers=list ) @@ -585,6 +667,7 @@ def test_configure_logging_uses_console_no_color_formatter_if_color_disabled(): multi_stream_handler.should_receive('setFormatter').with_args(fake_formatter).once() flexmock(module).should_receive('Multi_stream_handler').and_return(multi_stream_handler) + flexmock(module).should_receive('flush_delayed_logging') flexmock(module.logging).should_receive('basicConfig').with_args( level=logging.INFO, handlers=list ) From 9a9a8fd1c6b5684c63ef62e808fb8438e55829e5 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 7 Feb 2025 14:09:26 -0800 Subject: [PATCH 002/226] Add a "!credential" tag for loading systemd credentials into borgmatic configuration (#966). --- NEWS | 3 +++ borgmatic/commands/borgmatic.py | 29 ++++++++++++++------ borgmatic/config/credential.py | 33 +++++++++++++++++++++++ borgmatic/config/load.py | 20 +++++++++++++- borgmatic/config/validate.py | 14 +++++++--- borgmatic/hooks/credential/systemd.py | 39 +++++++++++++++++++++++++++ 6 files changed, 126 insertions(+), 12 deletions(-) create mode 100644 borgmatic/config/credential.py create mode 100644 borgmatic/hooks/credential/systemd.py diff --git a/NEWS b/NEWS index 0b96a621..f065dab9 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,7 @@ 1.9.10.dev0 + * #966: Add a "!credential" tag for loading systemd credentials into borgmatic configuration + files. See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/ * #987: Fix a "list" action error when the "encryption_passcommand" option is set. * #987: When both "encryption_passcommand" and "encryption_passphrase" are configured, prefer "encryption_passphrase" even if it's an empty value. diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 8f677191..f0d48cbe 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -533,15 +533,20 @@ def run_actions( ) -def load_configurations(config_filenames, overrides=None, resolve_env=True): +def load_configurations( + config_filenames, overrides=None, resolve_env=True, resolve_credentials=True +): ''' - Given a sequence of configuration filenames, load and validate each configuration file. Return - the results as a tuple of: dict of configuration filename to corresponding parsed configuration, - a sequence of paths for all loaded configuration files (including includes), and a sequence of - logging.LogRecord instances containing any parse errors. + Given a sequence of configuration filenames, a sequence of configuration file override strings + in the form of "option.suboption=value", whether to resolve environment variables, and whether + to resolve credentials, load and validate each configuration file. Return the results as a tuple + of: dict of configuration filename to corresponding parsed configuration, a sequence of paths + for all loaded configuration files (including includes), and a sequence of logging.LogRecord + instances containing any parse errors. Log records are returned here instead of being logged directly because logging isn't yet - initialized at this point! + initialized at this point! (Although with the Delayed_logging_handler now in place, maybe this + approach could change.) ''' # Dict mapping from config filename to corresponding parsed config dict. configs = collections.OrderedDict() @@ -563,7 +568,11 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True): ) try: configs[config_filename], paths, parse_logs = validate.parse_configuration( - config_filename, validate.schema_filename(), overrides, resolve_env + config_filename, + validate.schema_filename(), + overrides, + resolve_env, + resolve_credentials, ) config_paths.update(paths) logs.extend(parse_logs) @@ -907,9 +916,13 @@ def main(extra_summary_logs=[]): # pragma: no cover print(borgmatic.commands.completion.fish.fish_completion()) sys.exit(0) + validate = bool('validate' in arguments) config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths)) configs, config_paths, parse_logs = load_configurations( - config_filenames, global_arguments.overrides, global_arguments.resolve_env + config_filenames, + global_arguments.overrides, + resolve_env=global_arguments.resolve_env and not validate, + resolve_credentials=not validate, ) configuration_parse_errors = ( (max(log.levelno for log in parse_logs) >= logging.CRITICAL) if parse_logs else False diff --git a/borgmatic/config/credential.py b/borgmatic/config/credential.py new file mode 100644 index 00000000..77fc1a4f --- /dev/null +++ b/borgmatic/config/credential.py @@ -0,0 +1,33 @@ +import borgmatic.hooks.dispatch + + +def resolve_credentials(config, item=None): + ''' + Resolves values like "!credential hookname credentialname" from the given configuration by + calling relevant hooks to get the actual credential values. + + Raise ValueError if the config could not be parsed or the credential could not be loaded. + ''' + if not item: + item = config + + if isinstance(item, str): + if item.startswith('!credential '): + try: + (tag_name, hook_name, credential_name) = item.split(' ', 2) + except ValueError: + raise ValueError(f'Cannot load credential with invalid syntax "{item}"') + + return borgmatic.hooks.dispatch.call_hook( + 'load_credential', config, hook_name, credential_name + ) + + if isinstance(item, list): + for index, subitem in enumerate(item): + item[index] = resolve_credentials(config, subitem) + + if isinstance(item, dict): + for key, value in item.items(): + item[key] = resolve_credentials(config, value) + + return item diff --git a/borgmatic/config/load.py b/borgmatic/config/load.py index 978fc8e4..3285853b 100644 --- a/borgmatic/config/load.py +++ b/borgmatic/config/load.py @@ -69,7 +69,7 @@ def include_configuration(loader, filename_node, include_directory, config_paths ] raise ValueError( - '!include value is not supported; use a single filename or a list of filenames' + 'The value given for the !include tag is not supported; use a single filename or a list of filenames instead' ) @@ -104,6 +104,23 @@ def raise_omit_node_error(loader, node): ) +def reserialize_tag_node(loader, tag_node): + ''' + Given a ruamel.yaml loader and a node for a tag and value, convert the node back into a string + of the form "!tagname value" and return it. The idea is that downstream code, rather than this + file's YAML loading logic, should be responsible for interpreting this particular tag—since the + downstream code actually understands the meaning behind the tag. + + Raise ValueError if the tag node's value isn't a string. + ''' + if isinstance(tag_node.value, str): + return f'{tag_node.tag} {tag_node.value}' + + raise ValueError( + f'The value given for the {tag_node.tag} tag is not supported; use a string instead' + ) + + class Include_constructor(ruamel.yaml.SafeConstructor): ''' A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including @@ -122,6 +139,7 @@ class Include_constructor(ruamel.yaml.SafeConstructor): config_paths=config_paths, ), ) + self.add_constructor('!credential', reserialize_tag_node) # These are catch-all error handlers for tags that don't get applied and removed by # deep_merge_nodes() below. diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index dce99635..04b25724 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -5,6 +5,7 @@ import jsonschema import ruamel.yaml import borgmatic.config +import borgmatic.config.credential from borgmatic.config import constants, environment, load, normalize, override @@ -84,12 +85,15 @@ def apply_logical_validation(config_filename, parsed_configuration): ) -def parse_configuration(config_filename, schema_filename, overrides=None, resolve_env=True): +def parse_configuration( + config_filename, schema_filename, overrides=None, resolve_env=True, resolve_credentials=True +): ''' Given the path to a config filename in YAML format, the path to a schema filename in a YAML rendition of JSON Schema format, a sequence of configuration file override strings in the form - of "option.suboption=value", return the parsed configuration as a data structure of nested dicts - and lists corresponding to the schema. Example return value: + of "option.suboption=value", whether to resolve environment variables, and whether to resolve + credentials, return the parsed configuration as a data structure of nested dicts and lists + corresponding to the schema. Example return value: { 'source_directories': ['/home', '/etc'], @@ -118,12 +122,16 @@ def parse_configuration(config_filename, schema_filename, overrides=None, resolv if resolve_env: environment.resolve_env_variables(config) + if resolve_credentials: + borgmatic.config.credential.resolve_credentials(config) + logs = normalize.normalize(config_filename, config) try: validator = jsonschema.Draft7Validator(schema) except AttributeError: # pragma: no cover validator = jsonschema.Draft4Validator(schema) + validation_errors = tuple(validator.iter_errors(config)) if validation_errors: diff --git a/borgmatic/hooks/credential/systemd.py b/borgmatic/hooks/credential/systemd.py new file mode 100644 index 00000000..7aa76970 --- /dev/null +++ b/borgmatic/hooks/credential/systemd.py @@ -0,0 +1,39 @@ +import functools +import logging +import os +import re + +import borgmatic.config.paths +import borgmatic.execute + +logger = logging.getLogger(__name__) + + +CREDENTIAL_NAME_PATTERN = re.compile(r'^\w+$') + + +def load_credential(hook_config, config, credential_name): + ''' + Given the hook configuration dict, the configuration dict, and a credential name to load, read + the credential from the corresonding systemd credential file and return it. + + Raise ValueError if the systemd CREDENTIALS_DIRECTORY environment variable is not set, the + credential name is invalid, or the credential file cannot be read. + ''' + credentials_directory = os.environ.get('CREDENTIALS_DIRECTORY') + + if not credentials_directory: + raise ValueError( + f'Cannot load credential "{credential_name}" because the systemd CREDENTIALS_DIRECTORY environment variable is not set' + ) + + if not CREDENTIAL_NAME_PATTERN.match(credential_name): + raise ValueError(f'Cannot load invalid credential name "{credential_name}"') + + try: + with open(os.path.join(credentials_directory, credential_name)) as credential_file: + return credential_file.read().rstrip(os.linesep) + except (FileNotFoundError, OSError) as error: + logger.error(error) + + raise ValueError(f'Cannot load credential "{credential_name}" from file: {error.filename}') From c5abcc1fdf9a670e192a492595183be37585f953 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 7 Feb 2025 16:04:10 -0800 Subject: [PATCH 003/226] Add documentation for the "!credential" tag (#966). --- README.md | 13 ++++- docs/how-to/provide-your-passwords.md | 71 +++++++++++++++++++------- docs/static/pushover.png | Bin 6488 -> 9708 bytes docs/static/systemd.png | Bin 0 -> 9944 bytes 4 files changed, 64 insertions(+), 20 deletions(-) create mode 100644 docs/static/systemd.png diff --git a/README.md b/README.md index b57f68d4..4d0fdad9 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,8 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/). ## Integrations +### Data + PostgreSQL MySQL MariaDB @@ -65,6 +67,11 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/). Btrfs LVM rclone +BorgBase + + +### Monitoring + Healthchecks Uptime Kuma Cronitor @@ -76,7 +83,11 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/). Apprise Zabbix Sentry -BorgBase + + +### Credentials + +Sentry ## Getting started diff --git a/docs/how-to/provide-your-passwords.md b/docs/how-to/provide-your-passwords.md index 60aa33b7..c7c7b84b 100644 --- a/docs/how-to/provide-your-passwords.md +++ b/docs/how-to/provide-your-passwords.md @@ -50,52 +50,85 @@ once per borgmatic run. ### Using systemd service credentials -Borgmatic supports using encrypted [credentials](https://systemd.io/CREDENTIALS/). +borgmatic supports using encrypted [systemd +credentials](https://systemd.io/CREDENTIALS/). To use this feature, start by +saving your password as an encrypted credential to +`/etc/credstore.encrypted/borgmatic.pw`, e.g., -Save your password as an encrypted credential to `/etc/credstore.encrypted/borgmatic.pw`, e.g., - -``` -# systemd-ask-password -n | systemd-creds encrypt - /etc/credstore.encrypted/borgmatic.pw +```bash +systemd-ask-password -n | systemd-creds encrypt - /etc/credstore.encrypted/borgmatic.pw ``` -Then uncomment or use the following in your configuration file: +Then use the following in your configuration file: ```yaml -encryption_passcommand: "cat ${CREDENTIALS_DIRECTORY}/borgmatic.pw" +encryption_passphrase: !credential systemd borgmatic.pw ``` +Prior to version 1.9.10 You can +accomplish the same thing with this configuration: + +```yaml +encryption_passcommand: cat ${CREDENTIALS_DIRECTORY}/borgmatic.pw +``` Note that the name `borgmatic.pw` is hardcoded in the systemd service file. -To use multiple different passwords, save them as encrypted credentials to `/etc/credstore.encrypted/borgmatic/`, e.g., +You can use the `!credential` tag for any option value in a borgmatic +configuration file. So for example, use `!credential` to load systemd +credentials for database or monitoring passwords: +```yaml +postgresql_databases: + - name: invoices + username: postgres + password: !credential systemd borgmatic_db1 ``` -# mkdir /etc/credstore.encrypted/borgmatic -# systemd-ask-password -n | systemd-creds encrypt --name=borgmatic_backupserver1 - /etc/credstore.encrypted/borgmatic/backupserver1 -# systemd-ask-password -n | systemd-creds encrypt --name=borgmatic_pw2 - /etc/credstore.encrypted/borgmatic/pw2 + +But first you'll need to modify the borgmatic systemd service file to support +loading multiple credentials (assuming you need to load more than one or +anything not named `borgmatic.pw`). + +To do this, save each encrypted credentials to +`/etc/credstore.encrypted/borgmatic/`. E.g., + +```bash +mkdir /etc/credstore.encrypted/borgmatic +systemd-ask-password -n | systemd-creds encrypt --name=borgmatic_backupserver1 - /etc/credstore.encrypted/borgmatic/backupserver1 +systemd-ask-password -n | systemd-creds encrypt --name=borgmatic_pw2 - /etc/credstore.encrypted/borgmatic/pw2 ... ``` -Ensure that the file names, (e.g. `backupserver1`) match the corresponding part of -the `--name` option *after* the underscore (_), and that the part *before* +Ensure that the file names, (e.g. `backupserver1`) match the corresponding part +of the `--name` option *after* the underscore (_), and that the part *before* the underscore matches the directory name (e.g. `borgmatic`). Then, uncomment the appropriate line in the systemd service file: ``` -# systemctl edit borgmatic.service +systemctl edit borgmatic.service ... # Load multiple encrypted credentials. LoadCredentialEncrypted=borgmatic:/etc/credstore.encrypted/borgmatic/ ``` -Finally, use the following in your configuration file: +Finally, use something like the following in your borgmatic configuration file +for each option value you'd like to load from systemd: -``` -encryption_passcommand: "cat ${CREDENTIALS_DIRECTORY}/borgmatic_backupserver1" +```yaml +encryption_passphrase: !credential systemd borgmatic_backupserver1 ``` -Adjust `borgmatic_backupserver1` according to the name given to the credential -and the directory set in the service file. +Prior to version 1.9.10 Use the +following instead, but only for the `encryption_passcommand` option and +not other options: + +```yaml +encryption_passcommand: cat ${CREDENTIALS_DIRECTORY}/borgmatic_backupserver1 +``` + +Adjust `borgmatic_backupserver1` according to the name of the credential and the +directory set in the service file. + ### Environment variable interpolation diff --git a/docs/static/pushover.png b/docs/static/pushover.png index e0ef72a9c069e3d53a658102678966e83c757108..4d49fc73b02d7d5ae321c9fd8da2476bd52eed98 100644 GIT binary patch literal 9708 zcmeHLcQoA1*HRu9SAA}d(zVu=zxBsvL+8a+aEA=rl~K}4bx zq6HBxdguL;Jnj5`=e*}R=Y9WscF*kGnYs5fcRq9H?)N+4M%T5d$S;x;5D-x5Xlob~ z5D*Cfa>7|+;5VXltq7<-A}n-}IywRbXMqNwNk~FK2uK1%fcQ&J0NO-^!~`c+p+KPn zeqw0;E0tiR+FeZF5h01SaBArS!;P^SPz43HUs`UOx>VL#_;0OXU;+0(B! zAe|upUTf(X7=gruMZv=2GJri;Oi~UkCI^-ViAu?V#pNU<0d2yJKRr4@`l9td5I_k+ zsjD05sH=lq+|WprBZ7bcm*N|%sQuQ6A;i!mM^k{BiNW=qQR?Dz&ZtUq&3syi5VK4A zwtHj-7vmT%Xf;+&Omy4zH?~-slbNg19uT@-NMW$aZr+bmd&tb8@TJXbg>CuG1=^>k zxt5CORC=An<8>oy4qh;OhujC>2Zh3sESR{2(3QPft^4Vou4yyJ%iaC2m;}#|XiSla z>p#N<;gG#jGAs02mX>DW-Qm3Y0x>(W3lVMT6g^=I1#t(Y_s2n-!INq#lID1~!zvcOS{phMw1p0ja58VuV1suIF8k2%#L~^*!J! zzQy@Ot(crymdPmS%#V?w$GuM&s4b(MMuDI?I zzzfXhStllFnMH1F)pq0Gj`{9y8@%yX-aT~lSUgk^0zwdnf>>ZJ4D{t-XeVJP9BqdX z#yPp1>=%K8GR_4GyNker>=5=SXGPAHH+W7E3a-d$E@=QZa8XCxL1}xtAxyllLtx%_ zVX|;eWhHV2oE!k)gup^UI44JEj2upp^AuMOkWacrI6*t?3H7C;!Oqumf731JCg zu#hGWXlLem4`jOF?(1RVB< zy^Fh>ms2Z%?7wMZQOG~Z`kQYjBd5#x zt0RERyj)o%V$^3F~GzSlfD5Al96K071}rTpE(DfEAcQ3B zM4>{mVkcl=xENGI9F7EwBK`tl=!OEK66*L@uTG%g0F;ENI9ysv5+(!}15hL+Wu*Zq z38)ZET2@L_Mp8^h+D`No3J#OIhIVs;0^5mlg4!cQT%7Gs2Tll=Q#I02pjZR|`bQdu z{GNt|z%p{+pYET)8KB`Pq}TtZeR6m}3V=Sa<=Q9=Fu&Jn*UuAWf^hxW`q^|uogO6+ z==4y?L190afPs1-e(pDb^>YYz2kLB(0J6ugaQ(d>^!(W^M z{{DY{e(lBo=M*5&zmxnUe*dNGU%LJg1OG_*-{|_6u7AY9KT`fTy8h4TBLDMs3gHYq z1$hE5OOFKSx_}q0&qn$X&Ew_^|d-QGd zuy65TYyYTrVt;M-XnptS)Axggt)tDoqtPFS-!_kywvHCJ58uu1$9~wcs@w>Aw>`gk zG_Z7#-nV<_&8Bj}I(O>o^3GAw$ll=6LHpc+N7I&Q)|&pSjpgm5jlHAYgJa?6Yy9b} z`00J#^i}BVP0g37|ofgmW(1 zmKXv83i^{mNbod;8R#U#>KJH}%@ds^AQrBqZ9x$bFy!lKs6ucfOE&XX3$<~i^7PPn zLLw5%)tVCNIoxkijjW#)N7>Py_+xs>n8vbffkZZ5pY$7t^7|xhv5#Mv6$$8QX|+CL z5l6mH5{3OKP@~!&kIj4BJ1nBA<=y(IB*_JPS-~+-@>!lWk~h#m>#(uL>DgtmE$6Kt z&y}CvmP)fpvu=IkgL(6vvbc>tu&Y~`(yGCAl;z7umon;W_3_r?cg8=Zyy@K4=tXXC zH9Z>jgz?QCu4adJjVb#jecHdaRsB?D_mO(>s^EAlQ|g||?uH{ri=%oa_mcbY~Sojrb^r?$HbFY z_N58)c8zSz4n1;UU0U|;sfZ*qMBcQvSC%htXJ72Oo#U?Um14MMXCK9KGl=1-&3N*p zr1oXImUjnd_#rQonENg+EHd)1__Vluu7bB`wWlqu8H#r}mEenc?^TCgIgFUz^(!g; zxGg9i7FIg<XuivIn$QoOU^F`(5oYPEl z{22GNV6-}Q5GBXF>F=5Lg0;nQf6B40J7GA`wT$TD-7A<^9yvmD(XYmr1yM4x?l+q& zZ!SaRut67cgba)u470z{B@X|lntKq|8+Lii;#kt;LnevpEZ&;=Yofn!1KZWuL92m{ zd`7?4LweIIuw$ZhnOX-TzJQg|`wz-=Y4-$P%GiFlZ*05gEuixE{r(Aq_B3< z*T?tmqKeH;6$_U1L5t#?L2Wx(YBr{;lL28dc}9MEnQL;nVQEELHHymg?F);Qjvtjh z8U?kjA7C(QQi~;zW=gzMT1*xmcLaE(>mv=N6sEUq*~@b;@R)_(ws`Xi%irl%DffI~ zJdhVLDUfZtt&askuDZC0>^}yoHd@&2Wog(g) z)nHv;ud&mdq1}?b$C2S(CcAXgS>L{unS3MSL}@-1xuT^LW1& zhui;TwD)RX`iZ*k7qisIp$T7FZElX~4~BWC*3T(=R&CZq`Yx#E&bTQl-uQTrrvs=XZspdKGrTirDJC>G)JMRul6Xhg`E*+{i0qU_LcRfU2cPI@@_WWU?Y^% zdp8c-zk4P7@mI(!UOF#_a`2rTVS10yaV3R$w6l9`85N&w6UC6f@+fWVC@blbMufVK z|7`TLI%K)dx4oLRghFwC$nu01`&~|+Ae?H;Js&Cv&iHfB_bTyz)sEJ@AM?v&7UYbq z1>nzC+WacE6Y55?7L2XaI!=0>iNu$(9a(2y@3?lDv3BbivCwaMC)5}^$2JQu=w-yY zq7y3V`K!Or2Cc576LFqllSJCY8w7SVo6dH|3a)R8+D`at61d^k`g)$P6^;~)Ix;U! zL7&lnF1dSA^u;0mwpe7ukeFoZ{dW{oukP!1)Zh4YJzKCZj6I<(tIpZC4Y9fL&f4sa zqI?cqWhK`k6@Cy#woU|QU=y?yZXhc#ZA%|E%lB6b!nIQElc8TmgfDOW#udXuCB`D) z!E+1Ls9C}}B(T62u3M0vz?kZ{d1u4kteI6@5@b$XA34!@_QOzG;2T$>&bl1 zn$f?*X#H#=nkf;~S^Z-prpYuK=j(f;GoZ^lbL^2r}q7^DT#!n-1wbT8wy>8=`E zEUu~W3*NZFYWy&(WL{glFdMO)z_0W*x#Pxby{24gz7LEqywy6C6+O>&k1dYtE7%1m zZB*?p_A|tdFV7GbHU$wb#N|h5ekEDkKNdUPnU+?N*6AE&p$w_X%S!2_(2jY> zt9cUX#$SQUw-D)4Md+FAJM21jNH7XI-(kHL<0$(zGqClMTAT~0FZAQow|5Vk>3E-E z^M7D!4C0I}B?7jEi|e$SsLmd5Y-m$6rf)1$=MYIGd-$1Ux^)*F%3?ZLF{#X(l++{w z*MBQw6hsD>1*qmzWnL7#)aoGjT^sWx2Mcl=&b-kT zcct;HNLq6M|F-azMEpd00|V$u(5g80YZLoCBypkojrIPntpHvzakO8tI3Pji){UvP za?sht>BK2L?QE&i+7{8Ew^I3M8}JFd$IApyY!Uf1^Wnab7SZjEHH+sDh@+DNy*NY7 znxnl5x_u<9eBIu=C#584yM700k}_J6#j^1;SMn$HPxRli=Yx=tZswHS|CX4k%##qJ zn7&Lr-Ju*dk@r?W;?fUFMMD={)$;sc%0bl_uaqC3Z$~#KvUhOaD4^|XV*I(_5n3xY zw%<~E z4a3M~Y?Z(~XDe^STcHz~NSPrTQquwBw^hA-P2bWWRu+y5IZLaDZZ#^`$EZk{gf`xP zrM93J@?_k5O2@#^+jF@mljywkRboV~o@8Wm7O-_p{#Q(NDg6?!sH~t_RnaEfd-v=`!ZH-4vRV z?GuVx_12|(l)4t*l;#lr!-ymFRtO@vA)PUsQ?>DtHl_d2+!59I{)AWJG(jSANmR5!&lJ)-|b+q}3Nnj}l01b~=TE`7dBsAvZMxIA3dy z;^&vb6&k)EY|&cN$qNY@gG!(99OZgEm3EOs5fr8GR5JsCRaNa$=m$`okB)D2zN+G* zQSWBl3S#`ScPljALbEt$U@+Ot)(_+OY%JBscrJoj077C*3xTO4KRs>LUP6CCu3k$b z3fpv^ok*`?zMk`O=M87+_)AVQ-aCZH7~*=xSSusdsw@#_j)gHE4aHds&n_aPk5~&! zenF|iF?Ap1fr~Reoe4?<32Ux*yYK}*x8H*ZsEb(4bs2pMzh^{CM_OJAVy$~AU%k8u z_b@Q-l)FA0&~|YNT~o0WFHfF-JFgC&@pEoJXKHhk1 zRo!qQvWU+f3UJwSHcxiZ9ZRt*Az>Yin;KF z--(c@i7wlUYU@^RAJfG1X13;_4w%1QwvpAt!pPF;+O^NoojSy!&LY`-lO3%$p{p+- zYf0_AzBq+au1NYHotTipXi~}opNuvirI@Bc=uK0SJ+qlKZ7OI;{P-e@-e~?)O2ie* z@ti{^9{uJQdti@_hh%+gi(L&n&(9KWe5`xF*Ux6x=_85pgPJno z6Pu#jm(SlfYE1Y*xxp6xJadpvA@&975CZ%pTq#C~0>a&K(VU3Vp%Wdc64L76Yf#pp zp7ZgQ;1&T@(9cbxDPzdh5>vNp} z_mOc09phznhbWr!X#`g)Nhy2T;47q`$XegOQrS*+d6S`NV|-6TMnYW}uPBZ2l8)?m zh<6VrpGs^>+_`lBdx7CMyXk<`CUBR^7JIMH#FcGo27w%x5aV1>r#vKJV!1kXG4d+} z+b;K*dU^@XVh+7WI4S|~pmld4!Rs)1@Ubb-I}Mrey{Kn6eC3*4tG$typS;rDGXkkr zOK1V(huLrM?0@zycg_$df<06u9$YX!@2@cFMIQSbOtwJb=xkd7-}GEDC)>xvs;n;U z+fnZAwPy+t>6P_s!o;Y2_iVv&`3Y~+)^wE}E=>xTM!%F}M>-O}i=PA>Dtb-O_Iqoe zQLk9?+-maG9ZcbuvH~CIrbRq`N%5ePc`{s8ur}hLjghJ5Iq8u7r6-$eG`+FKJ0->E zZhh}WoA)9!)^xp8oyu%q9SSANoYQ{U!gP;MB;dPZue*@wfuUAxeAJE zmhAW11#YR&*7PR|6<28bxXSb zY;!x>uHi29|6iVXyQ4NpXFg1(Gid_l@y9dd@s^(XYWJv6o0v?on#sGPl`7Ze}e~ z^zTL-vTgL5B${ijJ~Mn4JATJp3l;CKA@%@jo7_pj5FQ;tR5vIYay;mt@j_S7O~2y# z6SwundfIk8++XN0`=)8Ti{Tn;cVala-VYy`7&OLPj17C>a^)${Sli-T;mC01f$`Ef zHST1XK19chm$eYMTA(NZEi$Su)_s}n4eIk(vmw2R+-~Aqb)O37Ah9I39|$@4DwQ2A z*YY$-=$s~kSddP~1iBDw7^urF57(Vrb7`5W*^eE6Ab%gLBQ!cI=p9VwalE@bejGAb zE?piJ!%LGQ(zhahH#Jsff)gd^c9{-h*JB&I=ZpY6Eix7BH_Hh4F(DZHZn8b;Huv^r zr2(c1VhXdZr5SkHjmDrt@~E8>2CORG_3By6n%o_uO66BHB6!7^1go(v>2LZzG_v|0 zpN(EaG&iH8)GNmQ`L|_k16@r+S;*^v` zRpZ;Q%WW%MkSQzDR_vz04EN6D5${K=)#EtvWd8dRJ~B#ReBeQBUhH&B_(88-#mkoh zcYCSK57hC8MIB~)VUNe1qrS&jNm}M{?D7>3pndBS@))CldY#Ifp-rP zVTkrOfGiJDrKF0`_@}a)`MvRb(SO9WM3l%4`Cv2UKk7!OM}z4y(F_VA5;)Gf7L`Su zNa=KJr3l7VKVOqhE8YP)Tn!59Ku=?OtZVkjB`6aW+&ouh3a2+1>Wj}p#Pn$RwTh{Y zzuMdA+8G1Uszee=&5z%_LzKB?%Ls3CV80CByVVaudnYl@>{C9o)WlAqshZRaUg;9R z6H>5cxg%f6x|X&4SZyx#U#{dPl_QHKs%FTI2oPSeGEP`Ac~5rd!{LqVUY9q`%}4FH zo`iMM$c;Zdkn+o}H@>MA@K|J#J5Dot^Yy)*Eia^DTGr6Fg|8MBDj&+b6~Bv|-$uoZ z>e4QKJh~E`F`W6ex^p_qDtn-CCc>~DUaWRL zK_E3TkGn}!wSQkHi22Yo>Q?e7$drWgp>hqD#aip)YP_?*%?gJAjbZbX$(7@Nodr&R z7J?^i)~;Kt_{R-bS+@9&UbF75i`cZYT^qe>`LWvHPm{nVpS|WNFDF5BsfdHAN7zx= zzsY!X1p4VqI@~kMmuqINn+sM{^1%nMJwN_b)H(dnXY{IB@g?!*GgrB@ZzdB;SzNw6 z(?|KLk$fJ^Nbft+=45}b-8b$1yQ=Qcu7#`90jj!JmkLm09>CMcwaQeF&ATJq#>iI` z;<-+hEq5`&h|%({`&Ca%C|eCkIk@sFMpM=;xr7lXcVP=%-b2Zy@7W{ucG(~0^p5kB zcX8-P97H}h@-5M~`OK{}-oE(scJdp4<_sVC?w*=HVT!KhNUhz+TQD4^XwoWI@pA>( z_ZDz@cjr!Wx{XEd=ZQC22;8T(XXv)_PV*poSbFTci)kAluF-9aUD_m@tGe+cved(I yIMum&aC{`=o*m5+2NM3D*N8uRN)<NjbgRif&T+9kev+x delta 6299 zcmZ{oWl)^avaSaUF2RDkGf05oZi7Q`cP9iJB-qEG!JPyG!67iX>mY->1QH-XkO09U zI0R14zE!vCR_)tAx>v1UPrqGV{c9bne;3e<)zO0B<51y%Kp=co6-8YT2-W!CISUKz zpMN#@?0*A;orrG{|ExT@XYOaEtyV4bPZra zRW+3`_i*S)I9XReSPcJ58mcPF=?5$ySsd7Iwxwwug0NK?Shn}kcUV@lU$PAmiT56V zUO8SGzn7Us9n!5}$+Wjao3B2h|DNT0Yi_`msRn@%{CC5Qx0{x!SlGvpo?p&hznH$Z z0Qh2L_;|YJgBOM!9Em<&SzRLURtyAQ$n``J+ak>1%A0VM2Ae#I(CER4XV(Q_4|27H<+qmiI`(n&?`^_uFu&#NGmn;zH z4R3|Cj@mp0Ao#M4)O`c)5GRIWhm`ADV}iAyx3rC+?!iIh-Tclmz3l z&el475$U)P*g`sC`zFQ+R0zD)1g=+837Bf)BS)L4WVF@>-767*k1APn^L*cYBUh;* zux{}ifiGQPnLXiX_3> zt+MF%FgGa*%36YG^SK@^wQo$gx{@8*A#k2ye?U)bI+s^&=A>9!((3dKV=t2kxi8$D z(6w&XZag{NdOe!ixuW7;Cli7UXEXG2izy9lXNVse8wGX~Ww&D-Q0-AL(~)Kk$~8>#3S9WB`6k zupZLZ*0Nmg&FgMH13FZ2?s2XWc!boY z!KW=6|6w1!ewnXAmqjQMz zT9N3W(gU~<1?lD96V!3*);Xas)x&x5>SlLQsk3f`aI1zV-kx+j0=_A95w^7$i*An* z94&4F>wU}g)`Tw830HDeoRzb-_e@U@d)Xx8rx%9#8oaA5D^=UJ$o8LWOFZ_S4Si#np5 zCBq&xX(;ARc%8^L1-0577=R038=YnLLS ztUFJ*JO{Inxl))MCK=1Ho_^jDh(RzU@WtCC1D!6h4FqM0t(&ZkN}Q*$jK&E!@Q{5v zz#~Q7?_+t$2T@_3T~O&T2D!If9xkup7i@g3zkHRhz5S@lxbD-vaMl{M;SFaZr5s8W z;Z=Wn@s8zIiH~%PT)Okyw7=8{DfULh0#S?K{tQ0*@$+j>NfI{N4zY-7ofLBi6NaX7 zk}}ogKnRt9h1Z_*SwWm<)Q@MT{JCb6l)$FJ@6wA{{0DBdwz(j?C&pc8ZJlxkXXT5x zoU+v~P}5i*td>7NH!S~_M-hqj^KG!|tC9c$u3FTH$d(Db_^ z`}Z}hz3jz72K;1dgjHjjwWQ<`Y_0cb)OTSk8!8dlx`b0Nf|Sdp>;1``Db{9DT&4jm zBfUh$kM>l`?^A?dvfU6kd>JGKOH4YkF@7D6*RSMh6;H%%>Ir0ZiBr$N<{6v3%1MsL z7E7<`s1!`@r7b4(OTw`!@?Qiv3V*irRXDBgrOc> zjzxALTpuLR923j@Me;as!DEAp80+>a{uWzrvlMtFZyCc)LNq%-4|Eplk#woXGI?#vf>GQBXwRaNE zSvmht(Yo(V+UOjAcz4$sE4n2`iz;}Ovh9WY25?-$hgOi^(qoBTLyAZ^;_jTc;d^}K zu~A3k*w*#s-2Ro;b||?5^!P(229U$OZhlG{R*yEW~`!sNws4L=Q8`y+#AeNIT>-bo|W=PO;wpsRuJ_#gB zQr|Ql(vOMZXxq7(I@HdRxUsax3$0=_H{briPEWu?EXgv~?d+Ik=m1~RLEm^yUu9^& zK=0J(oHm7k%>%_>IaE*G=`&lmvSq#G%3jh-2OI86rT(3u)N0^pB_B4d4dD-gdK+xC&9m{?flB0;Au3^fxaMA_-fw~PBK zy-^`$+Hm#!O>cSMcGvCXMgAI6OJ6NS>_Fgibr5{Qy{>gloSRs#ih<;5$zR2V52j!u zLd=$}q!`{XKZqSt(BbeIj}&Jp%P9B!)F2}`5$S2CJ?UH82rSmez1#OTio!$6=2o#m z+HF$1_BPIFiQ_$!LNOba4ua44^e>fNIt3FU7c~XV0<`jnT4fO)b7mQW!d{G?m7NOb zdJHRgX0pArgYPkh=G&$N!ICg_?U|4&P!t>eGhHN3fJ$|P?J6K8b4$;s@~SLgmpqXE z&2EECOo5ycz)v?(l!sY%64JChMzN zlWJ@)#g)fiXy-GcI6w3T1yi6#rR?Pj)Qzy(FwluwNvFPjlT0HrYb!WDI>g~}S$&e5 z@S2K$RE1DFz;yo|%s5RXxM5@scDa}E28RI$NP)`(#lf`L0AaI#Be$M!JJ_F_VX4)Gn`d*79deX14 zp?@&+ZM#^NZ*!!ZpKlm9pqV97lE50wXB2XBjFiDme;HRwm)kd}eWpClbBFD;!R{wq zhTa2!Wd>ii=Jh4J1Yrr;i-6=eQBlhxSV8pl8&iDpHpu;zC+@*HX0}gGV{<6Dl!Zl_N^A0->v?k%@1xk#jadFGoi`)|QN3M{T`E1k8elm?l`@GG z{Jp+ zhL#j`{EF{Srn4qE@ru}RCGz4pQeOa*Bv}~k{Sx^5&j2i&GDj%3>58#BY6H&VtSA3R zPNP)T+YXSFz}&i|?9=RhUcYC2Zr)U4lDwl?Xy70>fPN3^<>y>m@t*xbuZOR2B|_vy9=w#`_@SU-8c;G zYQI)`PoFiJaX{_eWJ6h;D24}10$SYs4=yM)OA2-pNji{Ti}j)xMTyDrRfddV7ZtZ# z=SH)%4USP{B~hR1=zVK-`f^QJUNH2S+ne?6^pWRY`lpGFzoi6+X`!{ zfI^*;qLSU^VU};4Rc&c10P}twUG(I5IP+YceWcbYJ&Ei@H&MLG=4?fKi6( zD8pL(f-0_m@C6FNS?leMScc_&b7hleUb}?Y_ugeQk!P0Tc1y`QxW(vYly2E?n~Z$J zH!u=hsc3;3G7EbR{mSYDR!H|SXLX{U(+548NcFg67TUcVpgG1^yWS7U$x!HPl1>X9 zb9>1ab+*w7l9f6gfPHs}+yU@&{o~*4W{$oRz{tlWk1?4Kn4$L{(f?PDA_p%lx7fdV z*{Im7XO(lVr9It?uZr+*&c2BM4mRyzdSSD8?2M@TKoJ2Rg^Lw0*_TA<)QB_Qq?Kp% z{-MZcw){V1Y})`Xg@~Rf!|q;`*xyu)k?NVi&lK$Zk^kwZqfQcFC6qnU!+e&|%r{Iw zqH*aB^J2TD3cvS)r(e>d5JfyN_y5YuC6$R`#ubBTe#$-4X@xtkVJE~KPd^vW5f4}1 zU)!a_;8HjG64EZzIyjd$`C6buj|XN^oac((8-+(RGUVMF>4oGnG93}i*=plQ^;>)P z19u2}g1oR8SA_x4aMl8kxh2-1R49s{M@2j8PLby1yEGZUE-wv^!j%t4Yd^z>Gyjiwt*D{`_-2?pFYYvPV_7!C%1&8K#l3Ppe(( z>}N)531#4;elQE&XSJg$#!vCNDE=rxik$MBmVzkVkLg2igYB-PwgSFLP@ zH%xyXn**XEVZ32f) zK8{p?;nA+L@G$5q*)uC*6GIcYHsgk z?U{yfw7npnM-$WVHzt>$`8ca6&?0hKimHWr2{1#*bNJKf1R{6%ejeSdd6D9m<*0nL z*d}Zh{U}c@*1_|(ngBT=0V~9_v<*R5!Bl>wfpNPC#p_#{bxJwq)^%|fy}i^3qmcj8 zIC^f%5qM+SZ(Z=mJqtY^4P7X;cm)X!3rk~Hp6@o^h;}N2eDXj`BUBQPx88%s1dO{%Osz#n<-_?QBrmd<5(3XjoH=# ztikf_n;g4pje@JcG=t@v%F~2SjsT5 zb|XWTlyn03N#&@Q3P$KLzS2J8y|++Mx$y2F@a1_Ywy6!^95~ihd<#&5;GlRf<*H_Akb%%g=GrDjw%#u%-JuUaBw8x1x7BBjZxQw18}? zSV3hmpn*F}CI&f6ki~aB`9vJ}Et!3k$`4!XdjM8gM*O_qj!#$v3N~c{wiHbVudzl! zc8HI9Z?pZM;rCp5%sOqm@PGY#^2WY3`W8>>&Qya`pNoaDgU>-HpLPyErKljLb8>g8eW9SY1Rjm}q6bCjE!Dg^$ zOBKE0S;J+K7X(;iiKc3=0oa!w+KFW#HL;OrqpL#sLvo^2@RO{|mQ(yd6PDkbJzn_m z?S<6ZdnL)E}zH+w3uQ)3uwTYE2)j9fQB0j z^3m5O;WZD|mk7ImJZ6HD$~6*JG3`39PA!>PpM3J;dz55Tx$Y@r9Q9+OK-Ic%S$9&F ziq(EqlG!2jLC|eO33!vq)A8mDNu$vgjr*#J!6>rZS^+Xy4}tUat|>-T*cb;O6cu_$ z^^L;4SLi1*m^VS}N~puX2f~_1s)c-0srH%GxV(~Ds>yvuwxNMvqJ{$gI#d!OvSXlg z&+7nDeilhAA$t`!tz0ufLU*$U`4aEvGwaG3EjBL-izyIHKyVq=?%q+oMdJDKohee! z4=%3G-=-XVDw{>&qfyCo47RBBqfb2dw9;k|ULc1-RNCHrn9`DmuyQ29-~(cT9@#c! zKi5Rwy}4E4AbxaOuSzlzT!l$-k0@0ABp?cKtsr(mP&D7xNZE)l<|z(|07q<4(&B52 z&0clQ-lt5U4~_{BvsdE#9*wWsjXCGJiI>N~?fFI0NLfdm4pAezgze+ROQ9X-T%*YM zp6mcau>Un3N2U|%VZE!~3zxIkYN(WGk;S_IiKcxhi(V2}G5Cmnt>}kn~brYD>Hw; zrVSF77ht2QvOQ__+2SvEua7FOaNv(eMB|Vs4UsoZtv(ihCzydih8odrFjjS+r{O_M zK$Hmx9IQ=k9iIvo$vdvDu+sS6h`pw4>zK}TJ&Ctc=4%1G(nE}x$2lVS z;v9(8?!V9~?~!EO+INOMZ#Wf=$`?etuU3~66})JI9#yLiJ@|7Mryk0H`jbF0QBgqj zzu80f*;B;LV0z=>gw((9dse8DG1T4$>L70C?eH&v1o#DncwX}J2=eQ{6cZPEDK03? z&Cf5+&p$<{KmPv!?jH6oj_>~efMSorq5nUEKF{{<5HKPp;`Lp1*w OkgAfFVvW3Y`2PT0huP`? diff --git a/docs/static/systemd.png b/docs/static/systemd.png new file mode 100644 index 0000000000000000000000000000000000000000..40d9adf264b35e6ea85f8e6e96af78c2d0b08d5e GIT binary patch literal 9944 zcmeHtcT|&0w|7t}f(7YKnsf{aB|xasL3#%%nocK>(2I&9C{>XTA|2^PK%|K@5dlS- z5I|7`=^#a<-3L6!bI$kOweEV?y7#}6@XYL)+50zp_Uzfu%!Io}23n`7*{DGv&}kiQ zbrTSX>>)s#QXK0otqL_kzP0pKJ%0U`sahd@I12W<+JkC7b*9S8C| zz(WVnhdhT58yWdg9tqIwf6#}02>rox*poQfG0`_)$yKp%dn zj=tIeb%^|DuBD@I#4RBv4i%G>1>~U;Fgd7%98`u|Tv`q)DF+n?xXB*>rO_eMytums zfO6YWP0dJ0O^qAx>5g@D!GJ)%X+ifDwOcQ7v|lyNq!^zqPM>8^ zrad-g@EoPoxb*w3+etko;hC0HRGD0oED6b&HN@2$^+bQ{V}l_Fg5R27Nsv?WTS!}st))WnXIrB6$K-R8agQ!U(?+yQTZ|^f z>qYrKkS%nQo2IdjG4HiWtM-hCEq7gI>7DP^watg$=3^?SI?2mmR&i6prOjbbry@3Q z?{>`fw;Q@&OkP!!X-4~*Jy!AF`*?k&-lhmDzQi9&e`zM-b6_MW)9$e`A#EAe>6iv; z!F{~!zAfr??PuKj72OCV`8U;Ee!QieLXpp zyQ>%i?T*BV`MTl{Ck&*Z?2AXBoH0ahBnIc`rpUWo-^|PHh*spagy}=|@oE?cM{R#k zjH$nY8Oq-o1xNELD^V->$^ihb7$Sn(*VV<1Am^*ddxR?o(1+C!UhX3nqO&5e72ro| z?w%NKDKRNAC|JYS(OZI7iJDu%6OENKQP=zf0w^i+IuMC?IS9nZ$4AUZQq0{G2N8$E z;Si_n=h%;!0WN)gIdylG*P(hk>WaLF{mY@WF-HJL0)di-!(ifI zq%>9rECrQ8g5eUf2(Xm2qy!2IMWT^V=-(g=JsknBM7aE2t3xO>00ozn7Dplx7%)a! z3I>*fOCZ6rGO|dpBwPk1D=r1cN=u0!L7`D{n(m&i2w*xLT@g481n-7BYB)5woQjc- zBCmwluU?O8j9d^zEFhrBtLNzE<@*<_nWHPll!!RArnroxEL=tcE-Ma|kp=qxi^&4x zNdP?Y5c4Q@9<>}!iySZ*z_5rzKLr4e>VdJysd-`$M0Za!cXt;>-rrI8ce6gQoX`j& zLLEWG0HD9)u-Tt+7z~w_`^#u@`tE2)tl$5{eYkkI6@E>*wj%-P-|wjE*NQU5c>F5; zD!MoxEhTR5qeUTyK>Zp50pX27AL$8T{c1uvAlz^mAbR}au7B1${&%hbhoPjUq%dHZ zv@{AVC4q&45f~Y1Fa~fWNhC%Vjg^7@6P@6WCHf#dF)BE~j)1KI_dI$-aSI(yl<+^f zeH<`{vj7+j43z*wW&U4-LH-ypLSP1zY4F9m% zVc7XU{QNN&|A##Q(EoPwAKCX`bNy?s|HuOW5%a&^^{=`9BMbaT%>R1V|C+g||2j@# z+<;w>4{%swBeA{%4q6mQT`l#aBNu2}t^PVdoy2Qf6F?vu`oo6|l#<2*R8kUk^ff5I z9isvrC*y4S0R@4WymZu6%zQ@|({Z-+*34moCm4;4{oQZ!HE>lzXe8rkFLOVi+1^%JBGpL} zaBCTaUd635nbp;dz}f%D1h*Jy7~*a>EypJxW&YJkHA*Urhf_ zJ*je*`sCcLu$G_`FV+q(BUb-RK3=*b1_E7jxEm8Baeax`EVFdKp#0Uf@&;~RT~lZ z4?R>N;q)5u>IF^=q2hE-Sd-D$Swx=YSpo0vjeBhox#AlO0ynJ)fuyoWUqZ+lwT+Ee zAD!PZ4mB}((4@7gHnU5Wz+4}{S1tf9;1w>(1l3n3w-;k!Xc*%}S#T70c+>4Uy_C?# zNY4A*#AqnHV^p!fSJsbU)+!24S&&NX^U>$8!YK<>MWW_t?7kllxAe5@&CIIbjxIUy+dLe};l zW#XNtvS%;z+*@A`{UUxjnt4*UpeH2k6eu`wwGcg2wo6=f{hatniq3ZXV$qY8_1I9e zN_4X=x%xEObS#_`h@JZzhyO#t~IT4@rXdE3X7bP*;6K_?{)9#Ue@hSNlkT> z5{c3@WzBMNaY7@V)OqjY-Gb~}#`HMc$6x!W(a|yYGPjXucTQbJOF(W;J?Asv%D=p@ z%|>dJWp-C6$g_D!A`e;{k9gfY1Rv>_)*!_TS3d1^3PwNb z^-t{?Oyr>hcO;kD*B=uNtjhNj5M14f)lzDbM}EPR^sg&l_Z!(7`TmfZw3*xeF0sT) zVbYnimh0eK94_Vur&g!UOSedImLM6vz)8-@%6@v7w1O-)HI zIX}?mi(y@o_jXqqXS-+|gYI!Rg9kquYn9c!h^FDzwt6dwl!f)-$ln6BRM-$PKqX zX!09H`7J}Aj8?Te+7rSM(aGBRj|0Mz5%=zGyEha+CE*tHs*8f~@e%Jobu*PTTwmr1 z;m&M9l*Z*vi2$lzWD*rx8A|*1-EXpmUds@&y4GPXc0$E-V!AhZpRoR!p z>G!6R*ynxIdFoc)%OlKz^JkXjE;2cP{#o+LXL-ALY=A@}-F06TVRW9X_L%O!+8L8J z6$Cy-@!9FpjA*~U#ooK`in$D5g(A3XmlQ+&QHH6qY=~}$L`rP>bdLu_j z)Kg8fMq<~7qBV!`V{4@Yk5b#8MU|D*=GwWh-9{SrH^kV`n(GG{ihF`JFdE$|0<(c~Wb`wSow=d(R0kqk zP4DW3{G@|nvtiClU3Fp2Ktb;jEF>KYpH6rwu1x}V+j zFYA4pyyikkej-(vl*FPO--VWl*PLduML}8af7_fkUPCSrX)LA)$h`h&IfLjZTVdVPB??~&W-E}qbHTAweEQD^os%l9oVWkdegum z*Zob#gXH1PLS&k19k#e-=HDM|qoZ_YIT>FP*Ut9jPRmXLEVC`~Yx1+`t@}P$nIj}mh{IPm9v!=cuKX}fXXDOML z=+})VHNT~=cem&)Z>*X}AG>*@FfM&#$@$Z8v?o)9EA@tH=Oe~WNmo;`GucG1Nc|3^ zGZr_qUT(7=yee`*k=`bFE#`tG*3enjW)+^@QQW^!k zH7_iBp|wQ8Ol{9%cEiIWfrDO0RW-15rr1xp@6*iM@?^jZY*mH(lL)dIg}i!GA%+tw zuh0*o7H`L{aFZ&SGs_!H*w=QxUMgc-Q*g+$8>jJ{-bn{uDr{_Q{CsjhjIvL!Z|*E< z4lf`bZ(E;hi=(a8qdUx1Uwdbt!YpNQ#lGz|^kqq%vP#)e zB``tDsy%B&QMz)#O@sWt6#2#={Of@?BLi=qZJlR`6b$DMPpubq!zc#Xxgh+nXy$A2 z+<)a8$`^Wb&1&_*(uf8J1H+e!&}vvUix%AsI&b$&NPq9+^5*8rObtsk4&xfClp z88ZUcYr_UB>jp15%xC47IXk%>ZW3gNKs&vin98`;(!~H{6E%jW;9Rfrg_W}g5xA93 z3x&Vlg3u&hq}mmw+QHY>)a2#n<>%&QC=+gK^Eo*=TUkdDQs?AMa3^AhW<7~O6uSNF zjPRMLs3@Jg!qW65^Ye2V{K{I3y!>Dc$~j7eftmU0Uc}bT#;}U8g2KY+NZ%L9Ll=NX zG+MUnLyJeH;@N=AqQGt2@YF|N#W4a0QD)|5C%eF6!ouyaGJL0#-}^qY9i{N0h6K-IQBwNM5FxKfr0ht>1eMH&r$qR($rnxj$WSW*kj@vFiH=N{8yG>>8yZh?NTBWC1W?a>UWRgx>P;CF> z@f{zR4;A8K0z9iaCR6Xxq_mS~U@#ba@HpM-ih<#CRSH8P9d!SCGUu+$%b1oCEv48T zzEbBm*PwVWecyKH$lGi)G;ATtvHmif4+Ohi%v0pTl+G{_m$XFnbZrPvwsF%liEYF!2_r6Dapkl@Byw@sYd&yd*G}TbQog_NS>ujq&dQ|X+-~N=!Y9E#GQ`*iyqliD#_6ZUz+m?`TQ-#L$FWnvVM!pImI7L*qQ`}6UiU!_AhU6Jz2hJI%4Ks~8Y;4@d>flQ7 z*Mt_^g{2Xl;RBX;NaSX8O*&^#dF2D135Ot4Q~J4LCC0$gqLLj;TQiEjWSM#7i@Jaq zRYpqTskD*qfk(uM+qa+PB~aAtdcRFyUKNW74?ksW-ja~dr?_0{H_>dFb@B3W91HJk z_N;EfCLVd=SVdq<6}G+1;%m*~!3TZ(%cd+14UO7GQe>2z`9n**^9QfXGm<;Nn+lOn z&qK8)ElJ~L0WdS~iJ_d&aEZIXv8$>lR76(#Q_7_VWbiioh^+Iyw(Ee`DmNUw%5G_; zjG-W(pnJ*gD$6v(f6f<7wYBD#GrW0FNGIvY6rz0FZ}|k0I{~6C1%s70+2VoGma&R zcOOsP9~~K)g$H~}J(UKJ!Q^6`ElZRMbGr%p@_703>n(#*>jROJ`MufHs4S#mhOB7A z6b6!=Kh*BvZ<($n&)R735=-CRoLwC(GN?|~?6I@=@+42@$g^M?r*B^p5)#J8wouGw zpTwnf1mXnc;Vb?6;L~o~rN=2=UBm4*^t>7UQpH&9Z`fOJYa5|QNkCRs9>nt9;*0>Qg;Uln=+y5>@4LgL~cwRf~pl22Tz4lJq84RX+y z1vbLU zMcvA~*u6z>v@_3WcY;TpwtLyD!`gZ_W=L0evD_2w;nB6=_^4Iu6#Em&5QF^qgm`{M zosORV&7^($m=3$~v@*NK7L;gAkY%xTbh>WfcTN0{acxNgJ)=3Jbi~FgD%cZAJ;Y&Z z!5TTJq$AwLQ>aD_HT7-^6jLC6q?$PZj#M&Gj`uZ=^U+Z!R}Cw=vw#J%v&HF|fbGaF zt?BJ1ILWX5br?+|LyZ`ByRz)fjHB}*e@ujn$$4i|hGkTS$3o0;JBAb9w z)fwb=Bl*osvVjXAH~%2-q59ad%;1?lwOg(Dc<15@9&b3M89B+}=RteKuxxf+=#!bi zw03(oc6LC=Zf zw{IJb&ui#1J2*OT%0^z`*v*@jGqJ7n85=lfxyt%wRA!2%V%J_p*XF(Q$3RQL2#2%p z&RctCj-QCqfk2v71oWj)5_svf8UZ9|gFYv$p@7H+s(jP0Px7$F76sKt?08c(Q!cpiSS8 z9M045P7OA2E2K=g=+xI-+n?FM4t%gK@R&*XRF7F@PIF3a^s~!-scihEQ)OTE`E@s| zCC^suPFynA-}svTLV2&xN}r_@46`-Y_u5zOqcr{$7QHk>I7vw|%sO{hePSOQkhyLP zmD%iSHE;I)j1}=%N2bM3-&70WY@cYgb8c3QOXF}0rASHsd_qr~??L3LeGOreGY$A{ z7j56z)OwoQvVIYvR}iZ&+d`WY_XfLPsVdZK2Rs{UoehsKN$OP;|mF|@WVF?}=VvJwPC!FODerlTX8w}`A$B%a;&a)Qw|AMta2s`$&Et!6 zYx#2wp;5O;k^HdGnbH+2?o(|=Wmg5cz6(bCsAE()6&idx796iF2FFl6(Q)K`!dP6f zWUTLRq+xPJjcggyOhu(~Rn?24hh(p5IFbf4xX6`&Wu)lMSXC2dPS&f*KvOCnpBn6@ zJ*nPFmVKTwQp}}PyR_SHFgCGIXnWMex(~}u#jBGebk}H5h?;_K4!H7D%-Fl|f<}d^ zxAIPngnzkEureJRhd29oxI@v&&-HS?lvHPJ$2FT4Ru1mezl(T!z+t@nJc?xd@?b6K z6iw7C|5Q6`&%I(Nm=tE+2ocG6vhJSNEcZA&^Zbw*rxEOSDCAe)7gE5gFm zvc1%lDp2bMK3MrNf8LL_4Kx5VvkD@c&^{lC Date: Sat, 8 Feb 2025 10:42:11 -0800 Subject: [PATCH 004/226] Add automated tests for the systemd credential hook (#966). --- borgmatic/config/credential.py | 4 +- borgmatic/config/load.py | 6 +- tests/integration/config/test_load.py | 14 +++ tests/integration/config/test_validate.py | 30 +++++ tests/unit/commands/test_borgmatic.py | 20 ++- tests/unit/config/test_credential.py | 127 ++++++++++++++++++++ tests/unit/config/test_load.py | 14 +++ tests/unit/hooks/credential/test_systemd.py | 51 ++++++++ 8 files changed, 259 insertions(+), 7 deletions(-) create mode 100644 tests/unit/config/test_credential.py create mode 100644 tests/unit/hooks/credential/test_systemd.py diff --git a/borgmatic/config/credential.py b/borgmatic/config/credential.py index 77fc1a4f..119f2bc9 100644 --- a/borgmatic/config/credential.py +++ b/borgmatic/config/credential.py @@ -4,7 +4,9 @@ import borgmatic.hooks.dispatch def resolve_credentials(config, item=None): ''' Resolves values like "!credential hookname credentialname" from the given configuration by - calling relevant hooks to get the actual credential values. + calling relevant hooks to get the actual credential values. The item parameter is used to + support recursing through the config hierarchy; it represents the current piece of config being + looked at. Raise ValueError if the config could not be parsed or the credential could not be loaded. ''' diff --git a/borgmatic/config/load.py b/borgmatic/config/load.py index 3285853b..7a4d75ee 100644 --- a/borgmatic/config/load.py +++ b/borgmatic/config/load.py @@ -69,7 +69,7 @@ def include_configuration(loader, filename_node, include_directory, config_paths ] raise ValueError( - 'The value given for the !include tag is not supported; use a single filename or a list of filenames instead' + 'The value given for the !include tag is invalid; use a single filename or a list of filenames instead' ) @@ -116,9 +116,7 @@ def reserialize_tag_node(loader, tag_node): if isinstance(tag_node.value, str): return f'{tag_node.tag} {tag_node.value}' - raise ValueError( - f'The value given for the {tag_node.tag} tag is not supported; use a string instead' - ) + raise ValueError(f'The value given for the {tag_node.tag} tag is invalid; use a string instead') class Include_constructor(ruamel.yaml.SafeConstructor): diff --git a/tests/integration/config/test_load.py b/tests/integration/config/test_load.py index e13c69c8..ab57db64 100644 --- a/tests/integration/config/test_load.py +++ b/tests/integration/config/test_load.py @@ -225,6 +225,20 @@ def test_load_configuration_merges_multiple_file_include(): assert config_paths == {'config.yaml', '/tmp/include1.yaml', '/tmp/include2.yaml', 'other.yaml'} +def test_load_configuration_passes_through_credential_tag(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os).should_receive('getcwd').and_return('/tmp') + flexmock(module.os.path).should_receive('isabs').and_return(False) + flexmock(module.os.path).should_receive('exists').and_return(True) + config_file = io.StringIO('key: !credential foo bar') + config_file.name = 'config.yaml' + builtins.should_receive('open').with_args('config.yaml').and_return(config_file) + config_paths = {'other.yaml'} + + assert module.load_configuration('config.yaml', config_paths) == {'key': '!credential foo bar'} + assert config_paths == {'config.yaml', 'other.yaml'} + + def test_load_configuration_with_retain_tag_merges_include_but_keeps_local_values(): builtins = flexmock(sys.modules['builtins']) flexmock(module.os).should_receive('getcwd').and_return('/tmp') diff --git a/tests/integration/config/test_validate.py b/tests/integration/config/test_validate.py index 9cd5c980..e5138d32 100644 --- a/tests/integration/config/test_validate.py +++ b/tests/integration/config/test_validate.py @@ -283,3 +283,33 @@ def test_parse_configuration_applies_normalization_after_environment_variable_in } assert config_paths == {'/tmp/config.yaml'} assert logs + + +def test_parse_configuration_interpolates_credentials(): + mock_config_and_schema( + ''' + source_directories: + - /home + + repositories: + - path: hostname.borg + + encryption_passphrase: !credential systemd mycredential + ''' + ) + flexmock(os.environ).should_receive('get').replace_with(lambda variable_name: '/var') + credential_stream = io.StringIO('password') + credential_stream.name = '/var/mycredential' + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('/var/mycredential').and_return(credential_stream) + + config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + + assert config == { + 'source_directories': ['/home'], + 'repositories': [{'path': 'hostname.borg'}], + 'encryption_passphrase': 'password', + 'bootstrap': {}, + } + assert config_paths == {'/tmp/config.yaml'} + assert logs == [] diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index e5e599e1..ce109565 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -1114,7 +1114,17 @@ def test_run_actions_runs_multiple_actions_in_argument_order(): ) -def test_load_configurations_collects_parsed_configurations_and_logs(): +@pytest.mark.parametrize( + 'resolve_env,resolve_credentials', + ( + (True, True), + (False, True), + (True, False), + ), +) +def test_load_configurations_collects_parsed_configurations_and_logs( + resolve_env, resolve_credentials +): configuration = flexmock() other_configuration = flexmock() test_expected_logs = [flexmock(), flexmock()] @@ -1123,7 +1133,13 @@ def test_load_configurations_collects_parsed_configurations_and_logs(): configuration, ['/tmp/test.yaml'], test_expected_logs ).and_return(other_configuration, ['/tmp/other.yaml'], other_expected_logs) - configs, config_paths, logs = tuple(module.load_configurations(('test.yaml', 'other.yaml'))) + configs, config_paths, logs = tuple( + module.load_configurations( + ('test.yaml', 'other.yaml'), + resolve_env=resolve_env, + resolve_credentials=resolve_credentials, + ) + ) assert configs == {'test.yaml': configuration, 'other.yaml': other_configuration} assert config_paths == ['/tmp/other.yaml', '/tmp/test.yaml'] diff --git a/tests/unit/config/test_credential.py b/tests/unit/config/test_credential.py new file mode 100644 index 00000000..71d906a2 --- /dev/null +++ b/tests/unit/config/test_credential.py @@ -0,0 +1,127 @@ +import pytest +from flexmock import flexmock + + +from borgmatic.config import credential as module + + +def test_resolve_credentials_ignores_string_without_credential_tag(): + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').never() + + module.resolve_credentials(config=flexmock(), item='!no credentials here') + + +def test_resolve_credentials_with_invalid_credential_tag_raises(): + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').never() + + with pytest.raises(ValueError): + module.resolve_credentials(config=flexmock(), item='!credential systemd') + + +def test_resolve_credentials_with_valid_credential_tag_loads_credential(): + config = flexmock() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( + 'load_credential', + config, + 'systemd', + 'mycredential', + ).and_return('result').once() + + assert ( + module.resolve_credentials(config=config, item='!credential systemd mycredential') + == 'result' + ) + + +def test_resolve_credentials_with_list_recurses_and_loads_credentials(): + config = flexmock() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( + 'load_credential', + config, + 'systemd', + 'mycredential', + ).and_return('result1').once() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( + 'load_credential', + config, + 'systemd', + 'othercredential', + ).and_return('result2').once() + + assert module.resolve_credentials( + config=config, + item=['!credential systemd mycredential', 'nope', '!credential systemd othercredential'], + ) == ['result1', 'nope', 'result2'] + + +def test_resolve_credentials_with_dict_recurses_and_loads_credentials(): + config = flexmock() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( + 'load_credential', + config, + 'systemd', + 'mycredential', + ).and_return('result1').once() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( + 'load_credential', + config, + 'systemd', + 'othercredential', + ).and_return('result2').once() + + assert module.resolve_credentials( + config=config, + item={ + 'a': '!credential systemd mycredential', + 'b': 'nope', + 'c': '!credential systemd othercredential', + }, + ) == {'a': 'result1', 'b': 'nope', 'c': 'result2'} + + +def test_resolve_credentials_with_list_of_dicts_recurses_and_loads_credentials(): + config = flexmock() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( + 'load_credential', + config, + 'systemd', + 'mycredential', + ).and_return('result1').once() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( + 'load_credential', + config, + 'systemd', + 'othercredential', + ).and_return('result2').once() + + assert module.resolve_credentials( + config=config, + item=[ + {'a': '!credential systemd mycredential', 'b': 'nope'}, + {'c': '!credential systemd othercredential'}, + ], + ) == [{'a': 'result1', 'b': 'nope'}, {'c': 'result2'}] + + +def test_resolve_credentials_with_dict_of_lists_recurses_and_loads_credentials(): + config = flexmock() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( + 'load_credential', + config, + 'systemd', + 'mycredential', + ).and_return('result1').once() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( + 'load_credential', + config, + 'systemd', + 'othercredential', + ).and_return('result2').once() + + assert module.resolve_credentials( + config=config, + item={ + 'a': ['!credential systemd mycredential', 'nope'], + 'b': ['!credential systemd othercredential'], + }, + ) == {'a': ['result1', 'nope'], 'b': ['result2']} diff --git a/tests/unit/config/test_load.py b/tests/unit/config/test_load.py index 4c1d221c..e1fe2c52 100644 --- a/tests/unit/config/test_load.py +++ b/tests/unit/config/test_load.py @@ -43,3 +43,17 @@ def test_probe_and_include_file_with_relative_path_and_missing_files_raises(): with pytest.raises(FileNotFoundError): module.probe_and_include_file('include.yaml', ['/etc', '/var'], config_paths=set()) + + +def test_reserialize_tag_node_turns_it_into_string(): + assert ( + module.reserialize_tag_node(loader=flexmock(), tag_node=flexmock(tag='!tag', value='value')) + == '!tag value' + ) + + +def test_reserialize_tag_node_with_invalid_value_raises(): + with pytest.raises(ValueError): + assert module.reserialize_tag_node( + loader=flexmock(), tag_node=flexmock(tag='!tag', value=['value']) + ) diff --git a/tests/unit/hooks/credential/test_systemd.py b/tests/unit/hooks/credential/test_systemd.py new file mode 100644 index 00000000..d8f0ff2c --- /dev/null +++ b/tests/unit/hooks/credential/test_systemd.py @@ -0,0 +1,51 @@ +import io +import sys + +from flexmock import flexmock +import pytest + +from borgmatic.hooks.credential import systemd as module + + +def test_load_credential_without_credentials_directory_raises(): + flexmock(module.os.environ).should_receive('get').with_args('CREDENTIALS_DIRECTORY').and_return( + None + ) + + with pytest.raises(ValueError): + module.load_credential(hook_config={}, config={}, credential_name='mycredential') + + +def test_load_credential_with_invalid_credential_name_raises(): + flexmock(module.os.environ).should_receive('get').with_args('CREDENTIALS_DIRECTORY').and_return( + '/var' + ) + + with pytest.raises(ValueError): + module.load_credential(hook_config={}, config={}, credential_name='../../my!@#$credential') + + +def test_load_credential_reads_named_credential_from_file(): + flexmock(module.os.environ).should_receive('get').with_args('CREDENTIALS_DIRECTORY').and_return( + '/var' + ) + credential_stream = io.StringIO('password') + credential_stream.name = '/var/mycredential' + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('/var/mycredential').and_return(credential_stream) + + assert ( + module.load_credential(hook_config={}, config={}, credential_name='mycredential') + == 'password' + ) + + +def test_load_credential_with_file_not_found_error_raises(): + flexmock(module.os.environ).should_receive('get').with_args('CREDENTIALS_DIRECTORY').and_return( + '/var' + ) + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('/var/mycredential').and_raise(FileNotFoundError) + + with pytest.raises(ValueError): + module.load_credential(hook_config={}, config={}, credential_name='mycredential') From 66abf38b39f77d13cdcfaea232e5970aa3aa6e43 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 8 Feb 2025 17:50:59 -0800 Subject: [PATCH 005/226] Add end-to-end tests for the systemd credential hook (#966). --- NEWS | 2 + borgmatic/config/credential.py | 7 +- borgmatic/hooks/credential/systemd.py | 6 +- borgmatic/logger.py | 7 +- docs/how-to/provide-your-passwords.md | 6 ++ .../hooks/credential/test_systemd.py | 71 +++++++++++++++++++ tests/unit/config/test_credential.py | 14 +++- tests/unit/hooks/credential/test_systemd.py | 2 +- tests/unit/test_logger.py | 20 +++++- 9 files changed, 121 insertions(+), 14 deletions(-) create mode 100644 tests/end-to-end/hooks/credential/test_systemd.py diff --git a/NEWS b/NEWS index f065dab9..a86324e6 100644 --- a/NEWS +++ b/NEWS @@ -10,6 +10,8 @@ refused to run checks in this situation. * #989: Fix the log message code to avoid using Python 3.10+ logging features. Now borgmatic will work with Python 3.9 again. + * Capture and delay any log records produced before logging is fully configured, so early log + records don't get lost. * Add support for Python 3.13. 1.9.9 diff --git a/borgmatic/config/credential.py b/borgmatic/config/credential.py index 119f2bc9..2c1621de 100644 --- a/borgmatic/config/credential.py +++ b/borgmatic/config/credential.py @@ -1,7 +1,10 @@ import borgmatic.hooks.dispatch -def resolve_credentials(config, item=None): +UNSPECIFIED = object() + + +def resolve_credentials(config, item=UNSPECIFIED): ''' Resolves values like "!credential hookname credentialname" from the given configuration by calling relevant hooks to get the actual credential values. The item parameter is used to @@ -10,7 +13,7 @@ def resolve_credentials(config, item=None): Raise ValueError if the config could not be parsed or the credential could not be loaded. ''' - if not item: + if item is UNSPECIFIED: item = config if isinstance(item, str): diff --git a/borgmatic/hooks/credential/systemd.py b/borgmatic/hooks/credential/systemd.py index 7aa76970..de3eedaa 100644 --- a/borgmatic/hooks/credential/systemd.py +++ b/borgmatic/hooks/credential/systemd.py @@ -1,11 +1,7 @@ -import functools import logging import os import re -import borgmatic.config.paths -import borgmatic.execute - logger = logging.getLogger(__name__) @@ -15,7 +11,7 @@ CREDENTIAL_NAME_PATTERN = re.compile(r'^\w+$') def load_credential(hook_config, config, credential_name): ''' Given the hook configuration dict, the configuration dict, and a credential name to load, read - the credential from the corresonding systemd credential file and return it. + the credential from the corresponding systemd credential file and return it. Raise ValueError if the systemd CREDENTIALS_DIRECTORY environment variable is not set, the credential name is invalid, or the credential file cannot be read. diff --git a/borgmatic/logger.py b/borgmatic/logger.py index 1fc0da52..07b4de76 100644 --- a/borgmatic/logger.py +++ b/borgmatic/logger.py @@ -269,6 +269,10 @@ class Delayed_logging_handler(logging.handlers.BufferingHandler): target handlers are actually set). It's useful for holding onto messages logged before logging is configured, ensuring those records eventually make their way to the relevant logging handlers. + + When flushing, don't forward log records to a target handler if the record's log level is below + that of the handler. This recreates the standard logging behavior of, say, logging.DEBUG records + getting suppressed if a handler's level is only set to logging.INFO. ''' def __init__(self): @@ -288,7 +292,8 @@ class Delayed_logging_handler(logging.handlers.BufferingHandler): for record in self.buffer: for target in self.targets: - target.handle(record) + if record.levelno >= target.level: + target.handle(record) self.buffer.clear() finally: diff --git a/docs/how-to/provide-your-passwords.md b/docs/how-to/provide-your-passwords.md index c7c7b84b..ba70c2d6 100644 --- a/docs/how-to/provide-your-passwords.md +++ b/docs/how-to/provide-your-passwords.md @@ -129,6 +129,12 @@ encryption_passcommand: cat ${CREDENTIALS_DIRECTORY}/borgmatic_backupserver1 Adjust `borgmatic_backupserver1` according to the name of the credential and the directory set in the service file. +Be aware that when using this systemd `!credential` feature, you can no longer +run borgmatic outside of its systemd service, as the credentials are only +available from within the context of that service. The one exception is +`borgmatic config validate`, which doesn't actually load any credentials and +should continue working anywhere. + ### Environment variable interpolation diff --git a/tests/end-to-end/hooks/credential/test_systemd.py b/tests/end-to-end/hooks/credential/test_systemd.py new file mode 100644 index 00000000..b4d43966 --- /dev/null +++ b/tests/end-to-end/hooks/credential/test_systemd.py @@ -0,0 +1,71 @@ +import json +import os +import shutil +import subprocess +import sys +import tempfile + + +def generate_configuration(config_path, repository_path): + ''' + Generate borgmatic configuration into a file at the config path, and update the defaults so as + to work for testing, including updating the source directories, injecting the given repository + path, and tacking on an encryption passphrase loaded from systemd. + ''' + subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' ')) + config = ( + open(config_path) + .read() + .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path) + .replace('- path: /mnt/backup', '') + .replace('label: local', '') + .replace('- /home/user/path with spaces', '') + .replace('- /home', f'- {config_path}') + .replace('- /etc', '') + .replace('- /var/log/syslog*', '') + + '\nencryption_passphrase: !credential systemd mycredential' + ) + config_file = open(config_path, 'w') + config_file.write(config) + config_file.close() + + +def test_borgmatic_command(): + # Create a Borg repository. + temporary_directory = tempfile.mkdtemp() + repository_path = os.path.join(temporary_directory, 'test.borg') + extract_path = os.path.join(temporary_directory, 'extract') + + original_working_directory = os.getcwd() + os.mkdir(extract_path) + os.chdir(extract_path) + + try: + config_path = os.path.join(temporary_directory, 'test.yaml') + generate_configuration(config_path, repository_path) + + credential_path = os.path.join(temporary_directory, 'mycredential') + with open(credential_path, 'w') as credential_file: + credential_file.write('test') + + subprocess.check_call( + f'borgmatic -v 2 --config {config_path} repo-create --encryption repokey'.split(' '), + env=dict(os.environ, **{'CREDENTIALS_DIRECTORY': temporary_directory}), + ) + + # Run borgmatic to generate a backup archive, and then list it to make sure it exists. + subprocess.check_call( + f'borgmatic --config {config_path}'.split(' '), + env=dict(os.environ, **{'CREDENTIALS_DIRECTORY': temporary_directory}), + ) + output = subprocess.check_output( + f'borgmatic --config {config_path} list --json'.split(' '), + env=dict(os.environ, **{'CREDENTIALS_DIRECTORY': temporary_directory}), + ).decode(sys.stdout.encoding) + parsed_output = json.loads(output) + + assert len(parsed_output) == 1 + assert len(parsed_output[0]['archives']) == 1 + finally: + os.chdir(original_working_directory) + shutil.rmtree(temporary_directory) diff --git a/tests/unit/config/test_credential.py b/tests/unit/config/test_credential.py index 71d906a2..378bc18b 100644 --- a/tests/unit/config/test_credential.py +++ b/tests/unit/config/test_credential.py @@ -1,14 +1,22 @@ import pytest from flexmock import flexmock - from borgmatic.config import credential as module -def test_resolve_credentials_ignores_string_without_credential_tag(): +def test_resolve_credentials_passes_through_string_without_credential_tag(): flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').never() - module.resolve_credentials(config=flexmock(), item='!no credentials here') + assert ( + module.resolve_credentials(config=flexmock(), item='!no credentials here') + == '!no credentials here' + ) + + +def test_resolve_credentials_passes_through_none(): + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').never() + + assert module.resolve_credentials(config=flexmock(), item=None) == None def test_resolve_credentials_with_invalid_credential_tag_raises(): diff --git a/tests/unit/hooks/credential/test_systemd.py b/tests/unit/hooks/credential/test_systemd.py index d8f0ff2c..97b2877f 100644 --- a/tests/unit/hooks/credential/test_systemd.py +++ b/tests/unit/hooks/credential/test_systemd.py @@ -1,8 +1,8 @@ import io import sys -from flexmock import flexmock import pytest +from flexmock import flexmock from borgmatic.hooks.credential import systemd as module diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index a1222862..dffaafc0 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -382,8 +382,8 @@ def test_delayed_logging_handler_flush_forwards_each_record_to_each_target(): handler = module.Delayed_logging_handler() flexmock(handler).should_receive('acquire') flexmock(handler).should_receive('release') - handler.targets = [flexmock(), flexmock()] - handler.buffer = [flexmock(), flexmock()] + handler.targets = [flexmock(level=logging.DEBUG), flexmock(level=logging.DEBUG)] + handler.buffer = [flexmock(levelno=logging.DEBUG), flexmock(levelno=logging.DEBUG)] handler.targets[0].should_receive('handle').with_args(handler.buffer[0]).once() handler.targets[1].should_receive('handle').with_args(handler.buffer[0]).once() handler.targets[0].should_receive('handle').with_args(handler.buffer[1]).once() @@ -394,6 +394,22 @@ def test_delayed_logging_handler_flush_forwards_each_record_to_each_target(): assert handler.buffer == [] +def test_delayed_logging_handler_flush_skips_forwarding_when_log_record_is_too_low_for_target(): + handler = module.Delayed_logging_handler() + flexmock(handler).should_receive('acquire') + flexmock(handler).should_receive('release') + handler.targets = [flexmock(level=logging.INFO), flexmock(level=logging.DEBUG)] + handler.buffer = [flexmock(levelno=logging.DEBUG), flexmock(levelno=logging.INFO)] + handler.targets[0].should_receive('handle').with_args(handler.buffer[0]).never() + handler.targets[1].should_receive('handle').with_args(handler.buffer[0]).once() + handler.targets[0].should_receive('handle').with_args(handler.buffer[1]).once() + handler.targets[1].should_receive('handle').with_args(handler.buffer[1]).once() + + handler.flush() + + assert handler.buffer == [] + + def test_flush_delayed_logging_without_handlers_does_not_raise(): root_logger = flexmock(handlers=[]) root_logger.should_receive('removeHandler') From 97fe1a2c503cc537a770ec0dd1470cce61a99a52 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 8 Feb 2025 19:28:03 -0800 Subject: [PATCH 006/226] Flake fixes (#966). --- borgmatic/config/credential.py | 1 - tests/unit/config/test_credential.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/borgmatic/config/credential.py b/borgmatic/config/credential.py index 2c1621de..ccec7ded 100644 --- a/borgmatic/config/credential.py +++ b/borgmatic/config/credential.py @@ -1,6 +1,5 @@ import borgmatic.hooks.dispatch - UNSPECIFIED = object() diff --git a/tests/unit/config/test_credential.py b/tests/unit/config/test_credential.py index 378bc18b..f6e16b20 100644 --- a/tests/unit/config/test_credential.py +++ b/tests/unit/config/test_credential.py @@ -16,7 +16,7 @@ def test_resolve_credentials_passes_through_string_without_credential_tag(): def test_resolve_credentials_passes_through_none(): flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').never() - assert module.resolve_credentials(config=flexmock(), item=None) == None + assert module.resolve_credentials(config=flexmock(), item=None) is None def test_resolve_credentials_with_invalid_credential_tag_raises(): From b7e3ee8277262bece5629730cb05cc692ce76b7d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 9 Feb 2025 11:12:40 -0800 Subject: [PATCH 007/226] Revamped the credentials to load them much closer to where they're used (#966). --- borgmatic/borg/environment.py | 8 +++++- borgmatic/commands/borgmatic.py | 14 ++++------ borgmatic/config/credential.py | 37 ------------------------- borgmatic/config/validate.py | 12 +++----- borgmatic/hooks/credential/systemd.py | 2 +- borgmatic/hooks/credential/tag.py | 29 +++++++++++++++++++ borgmatic/hooks/monitoring/ntfy.py | 13 +++++++-- borgmatic/hooks/monitoring/pagerduty.py | 9 +++++- borgmatic/hooks/monitoring/pushover.py | 10 +++++-- borgmatic/hooks/monitoring/zabbix.py | 13 +++++++-- 10 files changed, 83 insertions(+), 64 deletions(-) delete mode 100644 borgmatic/config/credential.py create mode 100644 borgmatic/hooks/credential/tag.py diff --git a/borgmatic/borg/environment.py b/borgmatic/borg/environment.py index 839d0887..082389ab 100644 --- a/borgmatic/borg/environment.py +++ b/borgmatic/borg/environment.py @@ -1,6 +1,7 @@ import os import borgmatic.borg.passcommand +import borgmatic.hooks.credential.tag OPTION_TO_ENVIRONMENT_VARIABLE = { 'borg_base_directory': 'BORG_BASE_DIR', @@ -14,6 +15,8 @@ OPTION_TO_ENVIRONMENT_VARIABLE = { 'temporary_directory': 'TMPDIR', } +CREDENTIAL_OPTIONS = {'encryption_passphrase'} + DEFAULT_BOOL_OPTION_TO_DOWNCASE_ENVIRONMENT_VARIABLE = { 'relocated_repo_access_is_ok': 'BORG_RELOCATED_REPO_ACCESS_IS_OK', 'unknown_unencrypted_repo_access_is_ok': 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK', @@ -37,7 +40,10 @@ def make_environment(config): for option_name, environment_variable_name in OPTION_TO_ENVIRONMENT_VARIABLE.items(): value = config.get(option_name) - if value: + if option_name in CREDENTIAL_OPTIONS: + value = borgmatic.hooks.credential.tag.resolve_credential(value) + + if value is not None: environment[environment_variable_name] = str(value) passphrase = borgmatic.borg.passcommand.get_passphrase_from_passcommand(config) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index f0d48cbe..bf444a7c 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -534,15 +534,15 @@ def run_actions( def load_configurations( - config_filenames, overrides=None, resolve_env=True, resolve_credentials=True + config_filenames, overrides=None, resolve_env=True ): ''' Given a sequence of configuration filenames, a sequence of configuration file override strings - in the form of "option.suboption=value", whether to resolve environment variables, and whether - to resolve credentials, load and validate each configuration file. Return the results as a tuple - of: dict of configuration filename to corresponding parsed configuration, a sequence of paths - for all loaded configuration files (including includes), and a sequence of logging.LogRecord - instances containing any parse errors. + in the form of "option.suboption=value", and whether to resolve environment variables, load and + validate each configuration file. Return the results as a tuple of: dict of configuration + filename to corresponding parsed configuration, a sequence of paths for all loaded configuration + files (including includes), and a sequence of logging.LogRecord instances containing any parse + errors. Log records are returned here instead of being logged directly because logging isn't yet initialized at this point! (Although with the Delayed_logging_handler now in place, maybe this @@ -572,7 +572,6 @@ def load_configurations( validate.schema_filename(), overrides, resolve_env, - resolve_credentials, ) config_paths.update(paths) logs.extend(parse_logs) @@ -922,7 +921,6 @@ def main(extra_summary_logs=[]): # pragma: no cover config_filenames, global_arguments.overrides, resolve_env=global_arguments.resolve_env and not validate, - resolve_credentials=not validate, ) configuration_parse_errors = ( (max(log.levelno for log in parse_logs) >= logging.CRITICAL) if parse_logs else False diff --git a/borgmatic/config/credential.py b/borgmatic/config/credential.py deleted file mode 100644 index ccec7ded..00000000 --- a/borgmatic/config/credential.py +++ /dev/null @@ -1,37 +0,0 @@ -import borgmatic.hooks.dispatch - -UNSPECIFIED = object() - - -def resolve_credentials(config, item=UNSPECIFIED): - ''' - Resolves values like "!credential hookname credentialname" from the given configuration by - calling relevant hooks to get the actual credential values. The item parameter is used to - support recursing through the config hierarchy; it represents the current piece of config being - looked at. - - Raise ValueError if the config could not be parsed or the credential could not be loaded. - ''' - if item is UNSPECIFIED: - item = config - - if isinstance(item, str): - if item.startswith('!credential '): - try: - (tag_name, hook_name, credential_name) = item.split(' ', 2) - except ValueError: - raise ValueError(f'Cannot load credential with invalid syntax "{item}"') - - return borgmatic.hooks.dispatch.call_hook( - 'load_credential', config, hook_name, credential_name - ) - - if isinstance(item, list): - for index, subitem in enumerate(item): - item[index] = resolve_credentials(config, subitem) - - if isinstance(item, dict): - for key, value in item.items(): - item[key] = resolve_credentials(config, value) - - return item diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 04b25724..e591a6c5 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -5,7 +5,6 @@ import jsonschema import ruamel.yaml import borgmatic.config -import borgmatic.config.credential from borgmatic.config import constants, environment, load, normalize, override @@ -86,14 +85,14 @@ def apply_logical_validation(config_filename, parsed_configuration): def parse_configuration( - config_filename, schema_filename, overrides=None, resolve_env=True, resolve_credentials=True + config_filename, schema_filename, overrides=None, resolve_env=True ): ''' Given the path to a config filename in YAML format, the path to a schema filename in a YAML rendition of JSON Schema format, a sequence of configuration file override strings in the form - of "option.suboption=value", whether to resolve environment variables, and whether to resolve - credentials, return the parsed configuration as a data structure of nested dicts and lists - corresponding to the schema. Example return value: + of "option.suboption=value", and whether to resolve environment variables, return the parsed + configuration as a data structure of nested dicts and lists corresponding to the schema. Example + return value: { 'source_directories': ['/home', '/etc'], @@ -122,9 +121,6 @@ def parse_configuration( if resolve_env: environment.resolve_env_variables(config) - if resolve_credentials: - borgmatic.config.credential.resolve_credentials(config) - logs = normalize.normalize(config_filename, config) try: diff --git a/borgmatic/hooks/credential/systemd.py b/borgmatic/hooks/credential/systemd.py index de3eedaa..0e7cdb47 100644 --- a/borgmatic/hooks/credential/systemd.py +++ b/borgmatic/hooks/credential/systemd.py @@ -30,6 +30,6 @@ def load_credential(hook_config, config, credential_name): with open(os.path.join(credentials_directory, credential_name)) as credential_file: return credential_file.read().rstrip(os.linesep) except (FileNotFoundError, OSError) as error: - logger.error(error) + logger.warning(error) raise ValueError(f'Cannot load credential "{credential_name}" from file: {error.filename}') diff --git a/borgmatic/hooks/credential/tag.py b/borgmatic/hooks/credential/tag.py new file mode 100644 index 00000000..baef0b77 --- /dev/null +++ b/borgmatic/hooks/credential/tag.py @@ -0,0 +1,29 @@ +import functools + +import borgmatic.hooks.dispatch + +IS_A_HOOK = False + + +@functools.cache +def resolve_credential(tag): + ''' + Given a configuration tag string like "!credential hookname credentialname", resolve it by + calling the relevant hook to get the actual credential value. If the given tag is not actually a + credential tag, then return the value unchanged. + + Cache the value so repeated calls to this function don't need to load the credential repeatedly. + + Raise ValueError if the config could not be parsed or the credential could not be loaded. + ''' + if tag and tag.startswith('!credential '): + try: + (tag_name, hook_name, credential_name) = tag.split(' ', 2) + except ValueError: + raise ValueError(f'Cannot load credential with invalid syntax "{tag}"') + + return borgmatic.hooks.dispatch.call_hook( + 'load_credential', {}, hook_name, credential_name + ) + + return tag diff --git a/borgmatic/hooks/monitoring/ntfy.py b/borgmatic/hooks/monitoring/ntfy.py index f56a6047..160b2518 100644 --- a/borgmatic/hooks/monitoring/ntfy.py +++ b/borgmatic/hooks/monitoring/ntfy.py @@ -2,6 +2,8 @@ import logging import requests +import borgmatic.hooks.credential.tag + logger = logging.getLogger(__name__) @@ -47,9 +49,14 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev 'X-Tags': state_config.get('tags'), } - username = hook_config.get('username') - password = hook_config.get('password') - access_token = hook_config.get('access_token') + try: + username = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('username')) + password = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('password')) + access_token = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('access_token')) + except ValueError as error: + logger.warning(f'Ntfy credential error: {error}') + return + auth = None if access_token is not None: diff --git a/borgmatic/hooks/monitoring/pagerduty.py b/borgmatic/hooks/monitoring/pagerduty.py index 6afc27c2..b2ff1153 100644 --- a/borgmatic/hooks/monitoring/pagerduty.py +++ b/borgmatic/hooks/monitoring/pagerduty.py @@ -5,6 +5,7 @@ import platform import requests +import borgmatic.hooks.credential.tag from borgmatic.hooks.monitoring import monitor logger = logging.getLogger(__name__) @@ -39,11 +40,17 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev if dry_run: return + try: + inegration_key = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('integration_key')) + except ValueError as error: + logger.warning(f'PagerDuty credential error: {error}') + return + hostname = platform.node() local_timestamp = datetime.datetime.now(datetime.timezone.utc).astimezone().isoformat() payload = json.dumps( { - 'routing_key': hook_config['integration_key'], + 'routing_key': integration_key, 'event_action': 'trigger', 'payload': { 'summary': f'backup failed on {hostname}', diff --git a/borgmatic/hooks/monitoring/pushover.py b/borgmatic/hooks/monitoring/pushover.py index 8211c32d..4f86ee0d 100644 --- a/borgmatic/hooks/monitoring/pushover.py +++ b/borgmatic/hooks/monitoring/pushover.py @@ -2,6 +2,8 @@ import logging import requests +import borgmatic.hooks.credential.tag + logger = logging.getLogger(__name__) @@ -32,8 +34,12 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev state_config = hook_config.get(state.name.lower(), {}) - token = hook_config.get('token') - user = hook_config.get('user') + try: + token = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('token')) + user = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('user')) + except ValueError as error: + logger.warning(f'Pushover credential error: {error}') + return logger.info(f'Updating Pushover{dry_run_label}') diff --git a/borgmatic/hooks/monitoring/zabbix.py b/borgmatic/hooks/monitoring/zabbix.py index 96b0a9da..f66c93a5 100644 --- a/borgmatic/hooks/monitoring/zabbix.py +++ b/borgmatic/hooks/monitoring/zabbix.py @@ -2,6 +2,8 @@ import logging import requests +import borgmatic.hooks.credential.tag + logger = logging.getLogger(__name__) @@ -34,10 +36,15 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev }, ) + try: + username = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('username')) + password = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('password')) + api_key = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('api_key')) + except ValueError as error: + logger.warning(f'Zabbix credential error: {error}') + return + server = hook_config.get('server') - username = hook_config.get('username') - password = hook_config.get('password') - api_key = hook_config.get('api_key') itemid = hook_config.get('itemid') host = hook_config.get('host') key = hook_config.get('key') From 49719dc3093a1783d41820e69d0686b82b6df9d3 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 9 Feb 2025 11:35:26 -0800 Subject: [PATCH 008/226] Load credentials from database hooks (#966). --- borgmatic/borg/environment.py | 2 +- borgmatic/commands/borgmatic.py | 4 +-- borgmatic/config/validate.py | 4 +-- borgmatic/hooks/credential/tag.py | 4 +-- borgmatic/hooks/data_source/mariadb.py | 35 +++++++++++++++++------ borgmatic/hooks/data_source/mongodb.py | 35 +++++++++++++++++------ borgmatic/hooks/data_source/mysql.py | 35 +++++++++++++++++------ borgmatic/hooks/data_source/postgresql.py | 28 +++++++++++++----- borgmatic/hooks/monitoring/ntfy.py | 12 ++++++-- borgmatic/hooks/monitoring/pagerduty.py | 4 ++- 10 files changed, 118 insertions(+), 45 deletions(-) diff --git a/borgmatic/borg/environment.py b/borgmatic/borg/environment.py index 082389ab..c58dcdbf 100644 --- a/borgmatic/borg/environment.py +++ b/borgmatic/borg/environment.py @@ -41,7 +41,7 @@ def make_environment(config): value = config.get(option_name) if option_name in CREDENTIAL_OPTIONS: - value = borgmatic.hooks.credential.tag.resolve_credential(value) + value = borgmatic.hooks.credential.tag.resolve_credential(value) if value is not None: environment[environment_variable_name] = str(value) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index bf444a7c..a55f18ab 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -533,9 +533,7 @@ def run_actions( ) -def load_configurations( - config_filenames, overrides=None, resolve_env=True -): +def load_configurations(config_filenames, overrides=None, resolve_env=True): ''' Given a sequence of configuration filenames, a sequence of configuration file override strings in the form of "option.suboption=value", and whether to resolve environment variables, load and diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index e591a6c5..c80cee4d 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -84,9 +84,7 @@ def apply_logical_validation(config_filename, parsed_configuration): ) -def parse_configuration( - config_filename, schema_filename, overrides=None, resolve_env=True -): +def parse_configuration(config_filename, schema_filename, overrides=None, resolve_env=True): ''' Given the path to a config filename in YAML format, the path to a schema filename in a YAML rendition of JSON Schema format, a sequence of configuration file override strings in the form diff --git a/borgmatic/hooks/credential/tag.py b/borgmatic/hooks/credential/tag.py index baef0b77..634ca84d 100644 --- a/borgmatic/hooks/credential/tag.py +++ b/borgmatic/hooks/credential/tag.py @@ -22,8 +22,6 @@ def resolve_credential(tag): except ValueError: raise ValueError(f'Cannot load credential with invalid syntax "{tag}"') - return borgmatic.hooks.dispatch.call_hook( - 'load_credential', {}, hook_name, credential_name - ) + return borgmatic.hooks.dispatch.call_hook('load_credential', {}, hook_name, credential_name) return tag diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index d309d9e5..fb072a31 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -5,6 +5,7 @@ import shlex import borgmatic.borg.pattern import borgmatic.config.paths +import borgmatic.hooks.credential.tag from borgmatic.execute import ( execute_command, execute_command_and_capture_output, @@ -45,7 +46,11 @@ def database_names_to_dump(database, extra_environment, dry_run): + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) - + (('--user', database['username']) if 'username' in database else ()) + + ( + ('--user', borgmatic.hooks.credential.tag.resolve_credential(database['username'])) + if 'username' in database + else () + ) + ('--skip-column-names', '--batch') + ('--execute', 'show schemas') ) @@ -96,7 +101,11 @@ def execute_dump_command( + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) - + (('--user', database['username']) if 'username' in database else ()) + + ( + ('--user', borgmatic.hooks.credential.tag.resolve_credential(database['username'])) + if 'username' in database + else () + ) + ('--databases',) + database_names + ('--result-file', dump_filename) @@ -152,7 +161,11 @@ def dump_data_sources( for database in databases: dump_path = make_dump_path(borgmatic_runtime_directory) - extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None + extra_environment = ( + {'MYSQL_PWD': borgmatic.hooks.credential.tag.resolve_credential(database['password'])} + if 'password' in database + else None + ) dump_database_names = database_names_to_dump(database, extra_environment, dry_run) if not dump_database_names: @@ -251,11 +264,13 @@ def restore_data_source_dump( port = str( connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')) ) - username = connection_params['username'] or data_source.get( - 'restore_username', data_source.get('username') + username = borgmatic.hooks.credential.tag.resolve_credential( + connection_params['username'] + or data_source.get('restore_username', data_source.get('username')) ) - password = connection_params['password'] or data_source.get( - 'restore_password', data_source.get('password') + password = borgmatic.hooks.credential.tag.resolve_credential( + connection_params['password'] + or data_source.get('restore_password', data_source.get('password')) ) mariadb_restore_command = tuple( @@ -274,7 +289,11 @@ def restore_data_source_dump( + (('--protocol', 'tcp') if hostname or port else ()) + (('--user', username) if username else ()) ) - extra_environment = {'MYSQL_PWD': password} if password else None + extra_environment = ( + {'MYSQL_PWD': borgmatic.hooks.credential.tag.resolve_credential(password)} + if password + else None + ) logger.debug(f"Restoring MariaDB database {data_source['name']}{dry_run_label}") if dry_run: diff --git a/borgmatic/hooks/data_source/mongodb.py b/borgmatic/hooks/data_source/mongodb.py index a1a006a9..098a2aa4 100644 --- a/borgmatic/hooks/data_source/mongodb.py +++ b/borgmatic/hooks/data_source/mongodb.py @@ -4,6 +4,7 @@ import shlex import borgmatic.borg.pattern import borgmatic.config.paths +import borgmatic.hooks.credential.tag from borgmatic.execute import execute_command, execute_command_with_processes from borgmatic.hooks.data_source import dump @@ -98,8 +99,26 @@ def build_dump_command(database, dump_filename, dump_format): + (('--out', shlex.quote(dump_filename)) if dump_format == 'directory' else ()) + (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ()) + (('--port', shlex.quote(str(database['port']))) if 'port' in database else ()) - + (('--username', shlex.quote(database['username'])) if 'username' in database else ()) - + (('--password', shlex.quote(database['password'])) if 'password' in database else ()) + + ( + ( + '--username', + shlex.quote( + borgmatic.hooks.credential.tag.resolve_credential(database['username']) + ), + ) + if 'username' in database + else () + ) + + ( + ( + '--password', + shlex.quote( + borgmatic.hooks.credential.tag.resolve_credential(database['password']) + ), + ) + if 'password' in database + else () + ) + ( ('--authenticationDatabase', shlex.quote(database['authentication_database'])) if 'authentication_database' in database @@ -198,11 +217,11 @@ def build_restore_command(extract_process, database, dump_filename, connection_p 'restore_hostname', database.get('hostname') ) port = str(connection_params['port'] or database.get('restore_port', database.get('port', ''))) - username = connection_params['username'] or database.get( - 'restore_username', database.get('username') + username = borgmatic.hooks.credential.tag.resolve_credential( + connection_params['username'] or database.get('restore_username', database.get('username')) ) - password = connection_params['password'] or database.get( - 'restore_password', database.get('password') + password = borgmatic.hooks.credential.tag.resolve_credential( + connection_params['password'] or database.get('restore_password', database.get('password')) ) command = ['mongorestore'] @@ -217,9 +236,9 @@ def build_restore_command(extract_process, database, dump_filename, connection_p if port: command.extend(('--port', str(port))) if username: - command.extend(('--username', username)) + command.extend(('--username', borgmatic.hooks.credential.tag.resolve_credential(username))) if password: - command.extend(('--password', password)) + command.extend(('--password', borgmatic.hooks.credential.tag.resolve_credential(password))) if 'authentication_database' in database: command.extend(('--authenticationDatabase', database['authentication_database'])) if 'restore_options' in database: diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index a477d2ac..00863c90 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -5,6 +5,7 @@ import shlex import borgmatic.borg.pattern import borgmatic.config.paths +import borgmatic.hooks.credential.tag from borgmatic.execute import ( execute_command, execute_command_and_capture_output, @@ -45,7 +46,11 @@ def database_names_to_dump(database, extra_environment, dry_run): + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) - + (('--user', database['username']) if 'username' in database else ()) + + ( + ('--user', borgmatic.hooks.credential.tag.resolve_credential(database['username'])) + if 'username' in database + else () + ) + ('--skip-column-names', '--batch') + ('--execute', 'show schemas') ) @@ -95,7 +100,11 @@ def execute_dump_command( + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) - + (('--user', database['username']) if 'username' in database else ()) + + ( + ('--user', borgmatic.hooks.credential.tag.resolve_credential(database['username'])) + if 'username' in database + else () + ) + ('--databases',) + database_names + ('--result-file', dump_filename) @@ -151,7 +160,11 @@ def dump_data_sources( for database in databases: dump_path = make_dump_path(borgmatic_runtime_directory) - extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None + extra_environment = ( + {'MYSQL_PWD': borgmatic.hooks.credential.tag.resolve_credential(database['password'])} + if 'password' in database + else None + ) dump_database_names = database_names_to_dump(database, extra_environment, dry_run) if not dump_database_names: @@ -250,11 +263,13 @@ def restore_data_source_dump( port = str( connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')) ) - username = connection_params['username'] or data_source.get( - 'restore_username', data_source.get('username') + username = borgmatic.hooks.credential.tag.resolve_credential( + connection_params['username'] + or data_source.get('restore_username', data_source.get('username')) ) - password = connection_params['password'] or data_source.get( - 'restore_password', data_source.get('password') + password = borgmatic.hooks.credential.tag.resolve_credential( + connection_params['password'] + or data_source.get('restore_password', data_source.get('password')) ) mysql_restore_command = tuple( @@ -273,7 +288,11 @@ def restore_data_source_dump( + (('--protocol', 'tcp') if hostname or port else ()) + (('--user', username) if username else ()) ) - extra_environment = {'MYSQL_PWD': password} if password else None + extra_environment = ( + {'MYSQL_PWD': borgmatic.hooks.credential.tag.resolve_credential(password)} + if password + else None + ) logger.debug(f"Restoring MySQL database {data_source['name']}{dry_run_label}") if dry_run: diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index 40c3e57b..cfacf867 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -7,6 +7,7 @@ import shlex import borgmatic.borg.pattern import borgmatic.config.paths +import borgmatic.hooks.credential.tag from borgmatic.execute import ( execute_command, execute_command_and_capture_output, @@ -33,11 +34,14 @@ def make_extra_environment(database, restore_connection_params=None): try: if restore_connection_params: - extra['PGPASSWORD'] = restore_connection_params.get('password') or database.get( - 'restore_password', database['password'] + extra['PGPASSWORD'] = borgmatic.hooks.credential.tag.resolve_credential( + restore_connection_params.get('password') + or database.get('restore_password', database['password']) ) else: - extra['PGPASSWORD'] = database['password'] + extra['PGPASSWORD'] = borgmatic.hooks.credential.tag.resolve_credential( + database['password'] + ) except (AttributeError, KeyError): pass @@ -82,7 +86,11 @@ def database_names_to_dump(database, extra_environment, dry_run): + ('--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only') + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) - + (('--username', database['username']) if 'username' in database else ()) + + ( + ('--username', borgmatic.hooks.credential.tag.resolve_credential(database['username'])) + if 'username' in database + else () + ) + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ()) ) logger.debug('Querying for "all" PostgreSQL databases to dump') @@ -174,7 +182,12 @@ def dump_data_sources( + (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ()) + (('--port', shlex.quote(str(database['port']))) if 'port' in database else ()) + ( - ('--username', shlex.quote(database['username'])) + ( + '--username', + shlex.quote( + borgmatic.hooks.credential.tag.resolve_credential(database['username']) + ), + ) if 'username' in database else () ) @@ -290,8 +303,9 @@ def restore_data_source_dump( port = str( connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')) ) - username = connection_params['username'] or data_source.get( - 'restore_username', data_source.get('username') + username = borgmatic.hooks.credential.tag.resolve_credential( + connection_params['username'] + or data_source.get('restore_username', data_source.get('username')) ) all_databases = bool(data_source['name'] == 'all') diff --git a/borgmatic/hooks/monitoring/ntfy.py b/borgmatic/hooks/monitoring/ntfy.py index 160b2518..d0bf1dee 100644 --- a/borgmatic/hooks/monitoring/ntfy.py +++ b/borgmatic/hooks/monitoring/ntfy.py @@ -50,9 +50,15 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev } try: - username = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('username')) - password = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('password')) - access_token = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('access_token')) + username = borgmatic.hooks.credential.tag.resolve_credential( + hook_config.get('username') + ) + password = borgmatic.hooks.credential.tag.resolve_credential( + hook_config.get('password') + ) + access_token = borgmatic.hooks.credential.tag.resolve_credential( + hook_config.get('access_token') + ) except ValueError as error: logger.warning(f'Ntfy credential error: {error}') return diff --git a/borgmatic/hooks/monitoring/pagerduty.py b/borgmatic/hooks/monitoring/pagerduty.py index b2ff1153..0edc4d96 100644 --- a/borgmatic/hooks/monitoring/pagerduty.py +++ b/borgmatic/hooks/monitoring/pagerduty.py @@ -41,7 +41,9 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev return try: - inegration_key = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('integration_key')) + inegration_key = borgmatic.hooks.credential.tag.resolve_credential( + hook_config.get('integration_key') + ) except ValueError as error: logger.warning(f'PagerDuty credential error: {error}') return From efdbee934a1e6865da0e9cd817dc1e2a99ff5976 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 9 Feb 2025 15:27:58 -0800 Subject: [PATCH 009/226] Update documentation to describe delayed !credential tag approach (#966). --- borgmatic/config/schema.yaml | 73 ++++++++++++++++----------- docs/how-to/provide-your-passwords.md | 34 ++++++++----- 2 files changed, 65 insertions(+), 42 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 33a6bbe2..5fee3890 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -250,7 +250,7 @@ properties: repositories that were initialized with passphrase/repokey/keyfile encryption. Quote the value if it contains punctuation, so it parses correctly. And backslash any quote or backslash literals as well. - Defaults to not set. + Defaults to not set. Supports the "!credential" tag. example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" checkpoint_interval: type: integer @@ -989,13 +989,13 @@ properties: Username with which to connect to the database. Defaults to the username of the current user. You probably want to specify the "postgres" superuser here when the - database name is "all". + database name is "all". Supports the "!credential" tag. example: dbuser restore_username: type: string description: | Username with which to restore the database. Defaults to - the "username" option. + the "username" option. Supports the "!credential" tag. example: dbuser password: type: string @@ -1003,13 +1003,14 @@ properties: Password with which to connect to the database. Omitting a password will only work if PostgreSQL is configured to trust the configured username without a password or you - create a ~/.pgpass file. + create a ~/.pgpass file. Supports the "!credential" tag. example: trustsome1 restore_password: type: string description: | Password with which to connect to the restore database. - Defaults to the "password" option. + Defaults to the "password" option. Supports the + "!credential" tag. example: trustsome1 no_owner: type: boolean @@ -1169,13 +1170,14 @@ properties: type: string description: | Username with which to connect to the database. Defaults - to the username of the current user. + to the username of the current user. Supports the + "!credential" tag. example: dbuser restore_username: type: string description: | Username with which to restore the database. Defaults to - the "username" option. + the "username" option. Supports the "!credential" tag. example: dbuser password: type: string @@ -1183,6 +1185,14 @@ properties: Password with which to connect to the database. Omitting a password will only work if MariaDB is configured to trust the configured username without a password. + Supports the "!credential" tag. + example: trustsome1 + restore_password: + type: string + description: | + Password with which to connect to the restore database. + Defaults to the "password" option. Supports the + "!credential" tag. example: trustsome1 mariadb_dump_command: type: string @@ -1201,12 +1211,6 @@ properties: run a specific mariadb version (e.g., one inside a running container). Defaults to "mariadb". example: docker exec mariadb_container mariadb - restore_password: - type: string - description: | - Password with which to connect to the restore database. - Defaults to the "password" option. - example: trustsome1 format: type: string enum: ['sql'] @@ -1295,13 +1299,14 @@ properties: type: string description: | Username with which to connect to the database. Defaults - to the username of the current user. + to the username of the current user. Supports the + "!credential" tag. example: dbuser restore_username: type: string description: | Username with which to restore the database. Defaults to - the "username" option. + the "username" option. Supports the "!credential" tag. example: dbuser password: type: string @@ -1309,12 +1314,14 @@ properties: Password with which to connect to the database. Omitting a password will only work if MySQL is configured to trust the configured username without a password. + Supports the "!credential" tag. example: trustsome1 restore_password: type: string description: | Password with which to connect to the restore database. - Defaults to the "password" option. + Defaults to the "password" option. Supports the + "!credential" tag. example: trustsome1 mysql_dump_command: type: string @@ -1451,25 +1458,28 @@ properties: type: string description: | Username with which to connect to the database. Skip it - if no authentication is needed. + if no authentication is needed. Supports the + "!credential" tag. example: dbuser restore_username: type: string description: | Username with which to restore the database. Defaults to - the "username" option. + the "username" option. Supports the "!credential" tag. example: dbuser password: type: string description: | Password with which to connect to the database. Skip it - if no authentication is needed. + if no authentication is needed. Supports the + "!credential" tag. example: trustsome1 restore_password: type: string description: | Password with which to connect to the restore database. - Defaults to the "password" option. + Defaults to the "password" option. Supports the + "!credential" tag. example: trustsome1 authentication_database: type: string @@ -1528,18 +1538,20 @@ properties: username: type: string description: | - The username used for authentication. + The username used for authentication. Supports the + "!credential" tag. example: testuser password: type: string description: | - The password used for authentication. + The password used for authentication. Supports the + "!credential" tag. example: fakepassword access_token: type: string description: | An ntfy access token to authenticate with instead of - username/password. + username/password. Supports the "!credential" tag. example: tk_AgQdq7mVBoFD37zQVN29RhuMzNIz2 start: type: object @@ -1634,14 +1646,15 @@ properties: token: type: string description: | - Your application's API token. + Your application's API token. Supports the "!credential" tag. example: 7ms6TXHpTokTou2P6x4SodDeentHRa user: type: string description: | - Your user/group key (or that of your target user), viewable - when logged into your dashboard: often referred to as + Your user/group key (or that of your target user), viewable + when logged into your dashboard: often referred to as USER_KEY in Pushover documentation and code examples. + Supports the "!credential" tag. example: hwRwoWsXMBWwgrSecfa9EfPey55WSN start: type: object @@ -1915,19 +1928,19 @@ properties: type: string description: | The username used for authentication. Not needed if using - an API key. + an API key. Supports the "!credential" tag. example: testuser password: type: string description: | The password used for authentication. Not needed if using - an API key. + an API key. Supports the "!credential" tag. example: fakepassword api_key: type: string description: | The API key used for authentication. Not needed if using - an username/password. + an username/password. Supports the "!credential" tag. example: fakekey start: type: object @@ -2208,7 +2221,7 @@ properties: type: string description: | PagerDuty integration key used to notify PagerDuty - when a backup errors. + when a backup errors. Supports the "!credential" tag. example: a177cad45bd374409f78906a810a3074 description: | Configuration for a monitoring integration with PagerDuty. Create an diff --git a/docs/how-to/provide-your-passwords.md b/docs/how-to/provide-your-passwords.md index ba70c2d6..6e908c78 100644 --- a/docs/how-to/provide-your-passwords.md +++ b/docs/how-to/provide-your-passwords.md @@ -71,11 +71,13 @@ accomplish the same thing with this configuration: ```yaml encryption_passcommand: cat ${CREDENTIALS_DIRECTORY}/borgmatic.pw ``` + Note that the name `borgmatic.pw` is hardcoded in the systemd service file. -You can use the `!credential` tag for any option value in a borgmatic -configuration file. So for example, use `!credential` to load systemd -credentials for database or monitoring passwords: +The `!credential` tag works for several different options in a borgmatic +configuration file besides just `encryption_passphrase`. For instance, the +username, password, and API token options within database and monitoring hooks +support `!credential`. For example: ```yaml postgresql_databases: @@ -84,11 +86,15 @@ postgresql_databases: password: !credential systemd borgmatic_db1 ``` -But first you'll need to modify the borgmatic systemd service file to support -loading multiple credentials (assuming you need to load more than one or -anything not named `borgmatic.pw`). +For specifics about which options are supported, see the +[configuration +reference](https://torsion.org/borgmatic/docs/reference/configuration/). -To do this, save each encrypted credentials to +To use these credentials, you'll need to modify the borgmatic systemd service +file to support loading multiple credentials (assuming you need to load more +than one or anything not named `borgmatic.pw`). + +Start by saving each encrypted credentials to `/etc/credstore.encrypted/borgmatic/`. E.g., ```bash @@ -129,11 +135,15 @@ encryption_passcommand: cat ${CREDENTIALS_DIRECTORY}/borgmatic_backupserver1 Adjust `borgmatic_backupserver1` according to the name of the credential and the directory set in the service file. -Be aware that when using this systemd `!credential` feature, you can no longer -run borgmatic outside of its systemd service, as the credentials are only -available from within the context of that service. The one exception is -`borgmatic config validate`, which doesn't actually load any credentials and -should continue working anywhere. +Be aware that when using this systemd `!credential` feature, you may no longer +be able to run certain borgmatic actions outside of the systemd service, as the +credentials are only available from within the context of that service. So for +instance, `borgmatic list` necessarily relies on the `encryption_passphrase` in +order to access the Borg repository, but it shouldn't need to load any +credentials for your database or monitoring hooks. + +The one exception is `borgmatic config validate`, which doesn't actually load +any credentials and should continue working anywhere. ### Environment variable interpolation From 775385e688e87390f6cf052ebfa23e644ffe02ba Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 9 Feb 2025 22:44:38 -0800 Subject: [PATCH 010/226] Get unit tests passing again (#966). --- borgmatic/borg/environment.py | 2 +- borgmatic/config/schema.yaml | 3 +- borgmatic/hooks/data_source/mariadb.py | 6 +- borgmatic/hooks/data_source/mongodb.py | 4 +- borgmatic/hooks/data_source/mysql.py | 6 +- borgmatic/hooks/monitoring/pagerduty.py | 2 +- tests/integration/config/test_validate.py | 30 ---- tests/unit/borg/test_environment.py | 4 + tests/unit/borg/test_passcommand.py | 21 +++ tests/unit/commands/test_borgmatic.py | 13 +- tests/unit/config/test_credential.py | 135 ------------------ tests/unit/hooks/credential/test_tag.py | 51 +++++++ tests/unit/hooks/data_source/test_mariadb.py | 66 +++++++++ tests/unit/hooks/data_source/test_mongodb.py | 36 +++++ tests/unit/hooks/data_source/test_mysql.py | 60 ++++++++ .../unit/hooks/data_source/test_postgresql.py | 90 ++++++++++++ tests/unit/hooks/monitoring/test_ntfy.py | 42 ++++++ tests/unit/hooks/monitoring/test_pagerduty.py | 18 +++ tests/unit/hooks/monitoring/test_pushover.py | 46 +++++- tests/unit/hooks/monitoring/test_zabbix.py | 84 +++++++++-- 20 files changed, 512 insertions(+), 207 deletions(-) delete mode 100644 tests/unit/config/test_credential.py create mode 100644 tests/unit/hooks/credential/test_tag.py diff --git a/borgmatic/borg/environment.py b/borgmatic/borg/environment.py index c58dcdbf..85033309 100644 --- a/borgmatic/borg/environment.py +++ b/borgmatic/borg/environment.py @@ -40,7 +40,7 @@ def make_environment(config): for option_name, environment_variable_name in OPTION_TO_ENVIRONMENT_VARIABLE.items(): value = config.get(option_name) - if option_name in CREDENTIAL_OPTIONS: + if option_name in CREDENTIAL_OPTIONS and value is not None: value = borgmatic.hooks.credential.tag.resolve_credential(value) if value is not None: diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 5fee3890..7c8e5b0a 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1646,7 +1646,8 @@ properties: token: type: string description: | - Your application's API token. Supports the "!credential" tag. + Your application's API token. Supports the "!credential" + tag. example: 7ms6TXHpTokTou2P6x4SodDeentHRa user: type: string diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index fb072a31..dcdd431f 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -289,11 +289,7 @@ def restore_data_source_dump( + (('--protocol', 'tcp') if hostname or port else ()) + (('--user', username) if username else ()) ) - extra_environment = ( - {'MYSQL_PWD': borgmatic.hooks.credential.tag.resolve_credential(password)} - if password - else None - ) + extra_environment = {'MYSQL_PWD': password} if password else None logger.debug(f"Restoring MariaDB database {data_source['name']}{dry_run_label}") if dry_run: diff --git a/borgmatic/hooks/data_source/mongodb.py b/borgmatic/hooks/data_source/mongodb.py index 098a2aa4..55dc6e9c 100644 --- a/borgmatic/hooks/data_source/mongodb.py +++ b/borgmatic/hooks/data_source/mongodb.py @@ -236,9 +236,9 @@ def build_restore_command(extract_process, database, dump_filename, connection_p if port: command.extend(('--port', str(port))) if username: - command.extend(('--username', borgmatic.hooks.credential.tag.resolve_credential(username))) + command.extend(('--username', username)) if password: - command.extend(('--password', borgmatic.hooks.credential.tag.resolve_credential(password))) + command.extend(('--password', password)) if 'authentication_database' in database: command.extend(('--authenticationDatabase', database['authentication_database'])) if 'restore_options' in database: diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index 00863c90..5be8b71e 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -288,11 +288,7 @@ def restore_data_source_dump( + (('--protocol', 'tcp') if hostname or port else ()) + (('--user', username) if username else ()) ) - extra_environment = ( - {'MYSQL_PWD': borgmatic.hooks.credential.tag.resolve_credential(password)} - if password - else None - ) + extra_environment = {'MYSQL_PWD': password} if password else None logger.debug(f"Restoring MySQL database {data_source['name']}{dry_run_label}") if dry_run: diff --git a/borgmatic/hooks/monitoring/pagerduty.py b/borgmatic/hooks/monitoring/pagerduty.py index 0edc4d96..53839eff 100644 --- a/borgmatic/hooks/monitoring/pagerduty.py +++ b/borgmatic/hooks/monitoring/pagerduty.py @@ -41,7 +41,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev return try: - inegration_key = borgmatic.hooks.credential.tag.resolve_credential( + integration_key = borgmatic.hooks.credential.tag.resolve_credential( hook_config.get('integration_key') ) except ValueError as error: diff --git a/tests/integration/config/test_validate.py b/tests/integration/config/test_validate.py index e5138d32..9cd5c980 100644 --- a/tests/integration/config/test_validate.py +++ b/tests/integration/config/test_validate.py @@ -283,33 +283,3 @@ def test_parse_configuration_applies_normalization_after_environment_variable_in } assert config_paths == {'/tmp/config.yaml'} assert logs - - -def test_parse_configuration_interpolates_credentials(): - mock_config_and_schema( - ''' - source_directories: - - /home - - repositories: - - path: hostname.borg - - encryption_passphrase: !credential systemd mycredential - ''' - ) - flexmock(os.environ).should_receive('get').replace_with(lambda variable_name: '/var') - credential_stream = io.StringIO('password') - credential_stream.name = '/var/mycredential' - builtins = flexmock(sys.modules['builtins']) - builtins.should_receive('open').with_args('/var/mycredential').and_return(credential_stream) - - config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') - - assert config == { - 'source_directories': ['/home'], - 'repositories': [{'path': 'hostname.borg'}], - 'encryption_passphrase': 'password', - 'bootstrap': {}, - } - assert config_paths == {'/tmp/config.yaml'} - assert logs == [] diff --git a/tests/unit/borg/test_environment.py b/tests/unit/borg/test_environment.py index e007664e..fda097b7 100644 --- a/tests/unit/borg/test_environment.py +++ b/tests/unit/borg/test_environment.py @@ -24,6 +24,10 @@ def test_make_environment_with_passphrase_should_set_environment(): ).and_return(None) flexmock(module.os).should_receive('pipe').never() flexmock(module.os.environ).should_receive('get').and_return(None) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) + environment = module.make_environment({'encryption_passphrase': 'pass'}) assert environment.get('BORG_PASSPHRASE') == 'pass' diff --git a/tests/unit/borg/test_passcommand.py b/tests/unit/borg/test_passcommand.py index 198cfd15..1a9a0e97 100644 --- a/tests/unit/borg/test_passcommand.py +++ b/tests/unit/borg/test_passcommand.py @@ -4,6 +4,7 @@ from borgmatic.borg import passcommand as module def test_run_passcommand_with_passphrase_configured_bails(): + module.run_passcommand.cache_clear() flexmock(module.borgmatic.execute).should_receive('execute_command_and_capture_output').never() assert ( @@ -13,6 +14,7 @@ def test_run_passcommand_with_passphrase_configured_bails(): def test_run_passcommand_without_passphrase_configured_executes_passcommand(): + module.run_passcommand.cache_clear() flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).and_return('passphrase').once() @@ -24,6 +26,7 @@ def test_run_passcommand_without_passphrase_configured_executes_passcommand(): def test_get_passphrase_from_passcommand_with_configured_passcommand_runs_it(): + module.run_passcommand.cache_clear() flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return( '/working' ) @@ -40,6 +43,7 @@ def test_get_passphrase_from_passcommand_with_configured_passcommand_runs_it(): def test_get_passphrase_from_passcommand_with_configured_passphrase_and_passcommand_detects_passphrase(): + module.run_passcommand.cache_clear() flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return( '/working' ) @@ -56,6 +60,7 @@ def test_get_passphrase_from_passcommand_with_configured_passphrase_and_passcomm def test_get_passphrase_from_passcommand_with_configured_blank_passphrase_and_passcommand_detects_passphrase(): + module.run_passcommand.cache_clear() flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return( '/working' ) @@ -69,3 +74,19 @@ def test_get_passphrase_from_passcommand_with_configured_blank_passphrase_and_pa ) is None ) + + +def test_run_passcommand_caches_passcommand_after_first_call(): + module.run_passcommand.cache_clear() + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return('passphrase').once() + + assert ( + module.run_passcommand('passcommand', passphrase_configured=False, working_directory=None) + == 'passphrase' + ) + assert ( + module.run_passcommand('passcommand', passphrase_configured=False, working_directory=None) + == 'passphrase' + ) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index ce109565..1615dcf4 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -1115,16 +1115,10 @@ def test_run_actions_runs_multiple_actions_in_argument_order(): @pytest.mark.parametrize( - 'resolve_env,resolve_credentials', - ( - (True, True), - (False, True), - (True, False), - ), + 'resolve_env', + ((True, False),), ) -def test_load_configurations_collects_parsed_configurations_and_logs( - resolve_env, resolve_credentials -): +def test_load_configurations_collects_parsed_configurations_and_logs(resolve_env): configuration = flexmock() other_configuration = flexmock() test_expected_logs = [flexmock(), flexmock()] @@ -1137,7 +1131,6 @@ def test_load_configurations_collects_parsed_configurations_and_logs( module.load_configurations( ('test.yaml', 'other.yaml'), resolve_env=resolve_env, - resolve_credentials=resolve_credentials, ) ) diff --git a/tests/unit/config/test_credential.py b/tests/unit/config/test_credential.py deleted file mode 100644 index f6e16b20..00000000 --- a/tests/unit/config/test_credential.py +++ /dev/null @@ -1,135 +0,0 @@ -import pytest -from flexmock import flexmock - -from borgmatic.config import credential as module - - -def test_resolve_credentials_passes_through_string_without_credential_tag(): - flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').never() - - assert ( - module.resolve_credentials(config=flexmock(), item='!no credentials here') - == '!no credentials here' - ) - - -def test_resolve_credentials_passes_through_none(): - flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').never() - - assert module.resolve_credentials(config=flexmock(), item=None) is None - - -def test_resolve_credentials_with_invalid_credential_tag_raises(): - flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').never() - - with pytest.raises(ValueError): - module.resolve_credentials(config=flexmock(), item='!credential systemd') - - -def test_resolve_credentials_with_valid_credential_tag_loads_credential(): - config = flexmock() - flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( - 'load_credential', - config, - 'systemd', - 'mycredential', - ).and_return('result').once() - - assert ( - module.resolve_credentials(config=config, item='!credential systemd mycredential') - == 'result' - ) - - -def test_resolve_credentials_with_list_recurses_and_loads_credentials(): - config = flexmock() - flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( - 'load_credential', - config, - 'systemd', - 'mycredential', - ).and_return('result1').once() - flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( - 'load_credential', - config, - 'systemd', - 'othercredential', - ).and_return('result2').once() - - assert module.resolve_credentials( - config=config, - item=['!credential systemd mycredential', 'nope', '!credential systemd othercredential'], - ) == ['result1', 'nope', 'result2'] - - -def test_resolve_credentials_with_dict_recurses_and_loads_credentials(): - config = flexmock() - flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( - 'load_credential', - config, - 'systemd', - 'mycredential', - ).and_return('result1').once() - flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( - 'load_credential', - config, - 'systemd', - 'othercredential', - ).and_return('result2').once() - - assert module.resolve_credentials( - config=config, - item={ - 'a': '!credential systemd mycredential', - 'b': 'nope', - 'c': '!credential systemd othercredential', - }, - ) == {'a': 'result1', 'b': 'nope', 'c': 'result2'} - - -def test_resolve_credentials_with_list_of_dicts_recurses_and_loads_credentials(): - config = flexmock() - flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( - 'load_credential', - config, - 'systemd', - 'mycredential', - ).and_return('result1').once() - flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( - 'load_credential', - config, - 'systemd', - 'othercredential', - ).and_return('result2').once() - - assert module.resolve_credentials( - config=config, - item=[ - {'a': '!credential systemd mycredential', 'b': 'nope'}, - {'c': '!credential systemd othercredential'}, - ], - ) == [{'a': 'result1', 'b': 'nope'}, {'c': 'result2'}] - - -def test_resolve_credentials_with_dict_of_lists_recurses_and_loads_credentials(): - config = flexmock() - flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( - 'load_credential', - config, - 'systemd', - 'mycredential', - ).and_return('result1').once() - flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( - 'load_credential', - config, - 'systemd', - 'othercredential', - ).and_return('result2').once() - - assert module.resolve_credentials( - config=config, - item={ - 'a': ['!credential systemd mycredential', 'nope'], - 'b': ['!credential systemd othercredential'], - }, - ) == {'a': ['result1', 'nope'], 'b': ['result2']} diff --git a/tests/unit/hooks/credential/test_tag.py b/tests/unit/hooks/credential/test_tag.py new file mode 100644 index 00000000..d802ce69 --- /dev/null +++ b/tests/unit/hooks/credential/test_tag.py @@ -0,0 +1,51 @@ +import pytest +from flexmock import flexmock + +from borgmatic.hooks.credential import tag as module + + +def test_resolve_credential_passes_through_string_without_credential_tag(): + module.resolve_credential.cache_clear() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').never() + + assert module.resolve_credential('!no credentials here') == '!no credentials here' + + +def test_resolve_credential_passes_through_none(): + module.resolve_credential.cache_clear() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').never() + + assert module.resolve_credential(None) is None + + +def test_resolve_credential_with_invalid_credential_tag_raises(): + module.resolve_credential.cache_clear() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').never() + + with pytest.raises(ValueError): + module.resolve_credential('!credential systemd') + + +def test_resolve_credential_with_valid_credential_tag_loads_credential(): + module.resolve_credential.cache_clear() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( + 'load_credential', + {}, + 'systemd', + 'mycredential', + ).and_return('result').once() + + assert module.resolve_credential('!credential systemd mycredential') == 'result' + + +def test_resolve_credential_caches_credential_after_first_call(): + module.resolve_credential.cache_clear() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( + 'load_credential', + {}, + 'systemd', + 'mycredential', + ).and_return('result').once() + + assert module.resolve_credential('!credential systemd mycredential') == 'result' + assert module.resolve_credential('!credential systemd mycredential') == 'result' diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py index e21979d6..f354e8d8 100644 --- a/tests/unit/hooks/data_source/test_mariadb.py +++ b/tests/unit/hooks/data_source/test_mariadb.py @@ -25,6 +25,9 @@ def test_database_names_to_dump_bails_for_dry_run(): def test_database_names_to_dump_queries_mariadb_for_database_names(): extra_environment = flexmock() + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('mariadb', '--skip-column-names', '--batch', '--execute', 'show schemas'), extra_environment=extra_environment, @@ -50,6 +53,9 @@ def test_dump_data_sources_dumps_each_database(): databases = [{'name': 'foo'}, {'name': 'bar'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( ('bar',) ) @@ -81,6 +87,9 @@ def test_dump_data_sources_dumps_with_password(): database = {'name': 'foo', 'username': 'root', 'password': 'trustsome1'} process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( ('bar',) ) @@ -108,6 +117,9 @@ def test_dump_data_sources_dumps_all_databases_at_once(): databases = [{'name': 'all'}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) flexmock(module).should_receive('execute_dump_command').with_args( database={'name': 'all'}, @@ -132,6 +144,9 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured databases = [{'name': 'all', 'format': 'sql'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) for name, process in zip(('foo', 'bar'), processes): @@ -199,6 +214,9 @@ def test_execute_dump_command_runs_mariadb_dump(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -231,6 +249,9 @@ def test_execute_dump_command_runs_mariadb_dump_without_add_drop_database(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -262,6 +283,9 @@ def test_execute_dump_command_runs_mariadb_dump_with_hostname_and_port(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -300,6 +324,9 @@ def test_execute_dump_command_runs_mariadb_dump_with_username_and_password(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -334,6 +361,9 @@ def test_execute_dump_command_runs_mariadb_dump_with_options(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -367,6 +397,9 @@ def test_execute_dump_command_runs_non_default_mariadb_dump_with_options(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -422,6 +455,9 @@ def test_execute_dump_command_with_duplicate_dump_skips_mariadb_dump(): def test_execute_dump_command_with_dry_run_skips_mariadb_dump(): flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').never() @@ -442,6 +478,9 @@ def test_execute_dump_command_with_dry_run_skips_mariadb_dump(): def test_dump_data_sources_errors_for_missing_all_databases(): databases = [{'name': 'all'}] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( 'databases/localhost/all' ) @@ -461,6 +500,9 @@ def test_dump_data_sources_errors_for_missing_all_databases(): def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run(): databases = [{'name': 'all'}] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( 'databases/localhost/all' ) @@ -483,6 +525,9 @@ def test_restore_data_source_dump_runs_mariadb_to_restore(): hook_config = [{'name': 'foo'}, {'name': 'bar'}] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mariadb', '--batch'), processes=[extract_process], @@ -511,6 +556,9 @@ def test_restore_data_source_dump_runs_mariadb_with_options(): hook_config = [{'name': 'foo', 'restore_options': '--harder'}] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mariadb', '--batch', '--harder'), processes=[extract_process], @@ -541,6 +589,9 @@ def test_restore_data_source_dump_runs_non_default_mariadb_with_options(): ] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ('custom_mariadb', '--batch', '--harder'), processes=[extract_process], @@ -569,6 +620,9 @@ def test_restore_data_source_dump_runs_mariadb_with_hostname_and_port(): hook_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mariadb', @@ -606,6 +660,9 @@ def test_restore_data_source_dump_runs_mariadb_with_username_and_password(): hook_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mariadb', '--batch', '--user', 'root'), processes=[extract_process], @@ -644,6 +701,9 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ ] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mariadb', @@ -695,6 +755,9 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ ] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mariadb', @@ -733,6 +796,9 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ def test_restore_data_source_dump_with_dry_run_skips_restore(): hook_config = [{'name': 'foo'}] + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').never() module.restore_data_source_dump( diff --git a/tests/unit/hooks/data_source/test_mongodb.py b/tests/unit/hooks/data_source/test_mongodb.py index 1d2c0c7b..1bc17e9a 100644 --- a/tests/unit/hooks/data_source/test_mongodb.py +++ b/tests/unit/hooks/data_source/test_mongodb.py @@ -124,6 +124,9 @@ def test_dump_data_sources_runs_mongodump_with_username_and_password(): flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( 'databases/localhost/foo' ) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -242,6 +245,9 @@ def test_dump_data_sources_runs_mongodumpall_for_all_databases(): def test_build_dump_command_with_username_injection_attack_gets_escaped(): database = {'name': 'test', 'username': 'bob; naughty-command'} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) command = module.build_dump_command(database, dump_filename='test', dump_format='archive') @@ -254,6 +260,9 @@ def test_restore_data_source_dump_runs_mongorestore(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ['mongorestore', '--archive', '--drop'], processes=[extract_process], @@ -285,6 +294,9 @@ def test_restore_data_source_dump_runs_mongorestore_with_hostname_and_port(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( [ 'mongorestore', @@ -330,6 +342,9 @@ def test_restore_data_source_dump_runs_mongorestore_with_username_and_password() flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( [ 'mongorestore', @@ -381,6 +396,9 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( [ 'mongorestore', @@ -436,6 +454,9 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( [ 'mongorestore', @@ -479,6 +500,9 @@ def test_restore_data_source_dump_runs_mongorestore_with_options(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ['mongorestore', '--archive', '--drop', '--harder'], processes=[extract_process], @@ -508,6 +532,9 @@ def test_restore_databases_dump_runs_mongorestore_with_schemas(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( [ 'mongorestore', @@ -545,6 +572,9 @@ def test_restore_data_source_dump_runs_psql_for_all_database_dump(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ['mongorestore', '--archive'], processes=[extract_process], @@ -573,6 +603,9 @@ def test_restore_data_source_dump_with_dry_run_skips_restore(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').never() module.restore_data_source_dump( @@ -596,6 +629,9 @@ def test_restore_data_source_dump_without_extract_process_restores_from_disk(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('/dump/path') + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ['mongorestore', '--dir', '/dump/path', '--drop'], processes=[], diff --git a/tests/unit/hooks/data_source/test_mysql.py b/tests/unit/hooks/data_source/test_mysql.py index 5eb46e35..e81182c7 100644 --- a/tests/unit/hooks/data_source/test_mysql.py +++ b/tests/unit/hooks/data_source/test_mysql.py @@ -16,6 +16,9 @@ def test_database_names_to_dump_passes_through_name(): def test_database_names_to_dump_bails_for_dry_run(): extra_environment = flexmock() + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_and_capture_output').never() names = module.database_names_to_dump({'name': 'all'}, extra_environment, dry_run=True) @@ -25,6 +28,9 @@ def test_database_names_to_dump_bails_for_dry_run(): def test_database_names_to_dump_queries_mysql_for_database_names(): extra_environment = flexmock() + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('mysql', '--skip-column-names', '--batch', '--execute', 'show schemas'), extra_environment=extra_environment, @@ -81,6 +87,9 @@ def test_dump_data_sources_dumps_with_password(): database = {'name': 'foo', 'username': 'root', 'password': 'trustsome1'} process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( ('bar',) ) @@ -199,6 +208,9 @@ def test_execute_dump_command_runs_mysqldump(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -231,6 +243,9 @@ def test_execute_dump_command_runs_mysqldump_without_add_drop_database(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -262,6 +277,9 @@ def test_execute_dump_command_runs_mysqldump_with_hostname_and_port(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -300,6 +318,9 @@ def test_execute_dump_command_runs_mysqldump_with_username_and_password(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -334,6 +355,9 @@ def test_execute_dump_command_runs_mysqldump_with_options(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -367,6 +391,9 @@ def test_execute_dump_command_runs_non_default_mysqldump(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -420,6 +447,9 @@ def test_execute_dump_command_with_duplicate_dump_skips_mysqldump(): def test_execute_dump_command_with_dry_run_skips_mysqldump(): flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').never() @@ -440,6 +470,9 @@ def test_execute_dump_command_with_dry_run_skips_mysqldump(): def test_dump_data_sources_errors_for_missing_all_databases(): databases = [{'name': 'all'}] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( 'databases/localhost/all' ) @@ -459,6 +492,9 @@ def test_dump_data_sources_errors_for_missing_all_databases(): def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run(): databases = [{'name': 'all'}] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( 'databases/localhost/all' ) @@ -481,6 +517,9 @@ def test_restore_data_source_dump_runs_mysql_to_restore(): hook_config = [{'name': 'foo'}, {'name': 'bar'}] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mysql', '--batch'), processes=[extract_process], @@ -509,6 +548,9 @@ def test_restore_data_source_dump_runs_mysql_with_options(): hook_config = [{'name': 'foo', 'restore_options': '--harder'}] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mysql', '--batch', '--harder'), processes=[extract_process], @@ -537,6 +579,9 @@ def test_restore_data_source_dump_runs_non_default_mysql_with_options(): hook_config = [{'name': 'foo', 'mysql_command': 'custom_mysql', 'restore_options': '--harder'}] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ('custom_mysql', '--batch', '--harder'), processes=[extract_process], @@ -565,6 +610,9 @@ def test_restore_data_source_dump_runs_mysql_with_hostname_and_port(): hook_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mysql', @@ -602,6 +650,9 @@ def test_restore_data_source_dump_runs_mysql_with_username_and_password(): hook_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mysql', '--batch', '--user', 'root'), processes=[extract_process], @@ -640,6 +691,9 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ ] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mysql', @@ -691,6 +745,9 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ ] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mysql', @@ -729,6 +786,9 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ def test_restore_data_source_dump_with_dry_run_skips_restore(): hook_config = [{'name': 'foo'}] + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').never() module.restore_data_source_dump( diff --git a/tests/unit/hooks/data_source/test_postgresql.py b/tests/unit/hooks/data_source/test_postgresql.py index 845859f7..05f3f7dd 100644 --- a/tests/unit/hooks/data_source/test_postgresql.py +++ b/tests/unit/hooks/data_source/test_postgresql.py @@ -24,6 +24,9 @@ def test_make_extra_environment_maps_options_to_environment(): 'PGSSLROOTCERT': 'root.crt', 'PGSSLCRL': 'crl.crl', } + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) extra_env = module.make_extra_environment(database) @@ -32,6 +35,9 @@ def test_make_extra_environment_maps_options_to_environment(): def test_make_extra_environment_with_cli_password_sets_correct_password(): database = {'name': 'foo', 'restore_password': 'trustsome1', 'password': 'anotherpassword'} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) extra = module.make_extra_environment( database, restore_connection_params={'password': 'clipassword'} @@ -78,6 +84,9 @@ def test_database_names_to_dump_passes_through_all_without_format(): def test_database_names_to_dump_with_all_and_format_and_dry_run_bails(): database = {'name': 'all', 'format': 'custom'} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_and_capture_output').never() assert module.database_names_to_dump(database, flexmock(), dry_run=True) == () @@ -85,6 +94,9 @@ def test_database_names_to_dump_with_all_and_format_and_dry_run_bails(): def test_database_names_to_dump_with_all_and_format_lists_databases(): database = {'name': 'all', 'format': 'custom'} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_and_capture_output').and_return( 'foo,test,\nbar,test,"stuff and such"' ) @@ -97,6 +109,9 @@ def test_database_names_to_dump_with_all_and_format_lists_databases(): def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostname_and_port(): database = {'name': 'all', 'format': 'custom', 'hostname': 'localhost', 'port': 1234} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ( 'psql', @@ -121,6 +136,9 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostnam def test_database_names_to_dump_with_all_and_format_lists_databases_with_username(): database = {'name': 'all', 'format': 'custom', 'username': 'postgres'} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ( 'psql', @@ -143,6 +161,9 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_usernam def test_database_names_to_dump_with_all_and_format_lists_databases_with_options(): database = {'name': 'all', 'format': 'custom', 'list_options': '--harder'} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('psql', '--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only', '--harder'), extra_environment=object, @@ -156,6 +177,9 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_options def test_database_names_to_dump_with_all_and_format_excludes_particular_databases(): database = {'name': 'all', 'format': 'custom'} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_and_capture_output').and_return( 'foo,test,\ntemplate0,test,blah' ) @@ -169,6 +193,9 @@ def test_database_names_to_dump_with_all_and_psql_command_uses_custom_command(): 'format': 'custom', 'psql_command': 'docker exec --workdir * mycontainer psql', } + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ( 'docker', @@ -219,6 +246,9 @@ def test_dump_data_sources_runs_pg_dump_for_each_database(): 'databases/localhost/foo' ).and_return('databases/localhost/bar') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') for name, process in zip(('foo', 'bar'), processes): @@ -323,6 +353,9 @@ def test_dump_data_sources_with_dry_run_skips_pg_dump(): 'databases/localhost/foo' ).and_return('databases/localhost/bar') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() @@ -349,6 +382,9 @@ def test_dump_data_sources_runs_pg_dump_with_hostname_and_port(): 'databases/database.example.org/foo' ) flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -394,6 +430,9 @@ def test_dump_data_sources_runs_pg_dump_with_username_and_password(): 'databases/localhost/foo' ) flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -437,6 +476,9 @@ def test_dump_data_sources_with_username_injection_attack_gets_escaped(): 'databases/localhost/foo' ) flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -477,6 +519,9 @@ def test_dump_data_sources_runs_pg_dump_with_directory_format(): 'databases/localhost/foo' ) flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_parent_directory_for_dump') flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() @@ -519,6 +564,9 @@ def test_dump_data_sources_runs_pg_dump_with_options(): 'databases/localhost/foo' ) flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -559,6 +607,9 @@ def test_dump_data_sources_runs_pg_dumpall_for_all_databases(): 'databases/localhost/all' ) flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -588,6 +639,9 @@ def test_dump_data_sources_runs_non_default_pg_dump(): 'databases/localhost/foo' ) flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -623,6 +677,9 @@ def test_restore_data_source_dump_runs_pg_restore(): hook_config = [{'name': 'foo', 'schemas': None}, {'name': 'bar'}] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') @@ -677,6 +734,9 @@ def test_restore_data_source_dump_runs_pg_restore_with_hostname_and_port(): ] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') @@ -739,6 +799,9 @@ def test_restore_data_source_dump_runs_pg_restore_with_username_and_password(): ] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('make_extra_environment').and_return( {'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'} ) @@ -810,6 +873,9 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ ] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('make_extra_environment').and_return( {'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'} ) @@ -889,6 +955,9 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ ] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('make_extra_environment').and_return( {'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'} ) @@ -962,6 +1031,9 @@ def test_restore_data_source_dump_runs_pg_restore_with_options(): ] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') @@ -1016,6 +1088,9 @@ def test_restore_data_source_dump_runs_psql_for_all_database_dump(): hook_config = [{'name': 'all', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') @@ -1055,6 +1130,9 @@ def test_restore_data_source_dump_runs_psql_for_plain_database_dump(): hook_config = [{'name': 'foo', 'format': 'plain', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') @@ -1106,6 +1184,9 @@ def test_restore_data_source_dump_runs_non_default_pg_restore_and_psql(): ] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') @@ -1167,6 +1248,9 @@ def test_restore_data_source_dump_runs_non_default_pg_restore_and_psql(): def test_restore_data_source_dump_with_dry_run_skips_restore(): hook_config = [{'name': 'foo', 'schemas': None}] + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') @@ -1191,6 +1275,9 @@ def test_restore_data_source_dump_with_dry_run_skips_restore(): def test_restore_data_source_dump_without_extract_process_restores_from_disk(): hook_config = [{'name': 'foo', 'schemas': None}] + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('/dump/path') @@ -1243,6 +1330,9 @@ def test_restore_data_source_dump_without_extract_process_restores_from_disk(): def test_restore_data_source_dump_with_schemas_restores_schemas(): hook_config = [{'name': 'foo', 'schemas': ['bar', 'baz']}] + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('/dump/path') diff --git a/tests/unit/hooks/monitoring/test_ntfy.py b/tests/unit/hooks/monitoring/test_ntfy.py index a5819ec9..efbecf96 100644 --- a/tests/unit/hooks/monitoring/test_ntfy.py +++ b/tests/unit/hooks/monitoring/test_ntfy.py @@ -36,6 +36,9 @@ def return_default_message_headers(state=Enum): def test_ping_monitor_minimal_config_hits_hosted_ntfy_on_fail(): hook_config = {'topic': topic} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.FAIL), @@ -57,6 +60,9 @@ def test_ping_monitor_with_access_token_hits_hosted_ntfy_on_fail(): 'topic': topic, 'access_token': 'abc123', } + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.FAIL), @@ -80,6 +86,9 @@ def test_ping_monitor_with_username_password_and_access_token_ignores_username_p 'password': 'fakepassword', 'access_token': 'abc123', } + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.FAIL), @@ -103,6 +112,9 @@ def test_ping_monitor_with_username_password_hits_hosted_ntfy_on_fail(): 'username': 'testuser', 'password': 'fakepassword', } + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.FAIL), @@ -121,6 +133,9 @@ def test_ping_monitor_with_username_password_hits_hosted_ntfy_on_fail(): def test_ping_monitor_with_password_but_no_username_warns(): hook_config = {'topic': topic, 'password': 'fakepassword'} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.FAIL), @@ -140,6 +155,9 @@ def test_ping_monitor_with_password_but_no_username_warns(): def test_ping_monitor_with_username_but_no_password_warns(): hook_config = {'topic': topic, 'username': 'testuser'} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.FAIL), @@ -159,6 +177,9 @@ def test_ping_monitor_with_username_but_no_password_warns(): def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_start(): hook_config = {'topic': topic} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').never() module.ping_monitor( @@ -173,6 +194,9 @@ def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_start(): def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_finish(): hook_config = {'topic': topic} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').never() module.ping_monitor( @@ -187,6 +211,9 @@ def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_finish(): def test_ping_monitor_minimal_config_hits_selfhosted_ntfy_on_fail(): hook_config = {'topic': topic, 'server': custom_base_url} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').with_args( f'{custom_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.FAIL), @@ -205,6 +232,9 @@ def test_ping_monitor_minimal_config_hits_selfhosted_ntfy_on_fail(): def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_fail_dry_run(): hook_config = {'topic': topic} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').never() module.ping_monitor( @@ -219,6 +249,9 @@ def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_fail_dry_run(): def test_ping_monitor_custom_message_hits_hosted_ntfy_on_fail(): hook_config = {'topic': topic, 'fail': custom_message_config} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=custom_message_headers, auth=None ).and_return(flexmock(ok=True)).once() @@ -235,6 +268,9 @@ def test_ping_monitor_custom_message_hits_hosted_ntfy_on_fail(): def test_ping_monitor_custom_state_hits_hosted_ntfy_on_start(): hook_config = {'topic': topic, 'states': ['start', 'fail']} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.START), @@ -253,6 +289,9 @@ def test_ping_monitor_custom_state_hits_hosted_ntfy_on_start(): def test_ping_monitor_with_connection_error_logs_warning(): hook_config = {'topic': topic} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.FAIL), @@ -272,6 +311,9 @@ def test_ping_monitor_with_connection_error_logs_warning(): def test_ping_monitor_with_other_error_logs_warning(): hook_config = {'topic': topic} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) response = flexmock(ok=False) response.should_receive('raise_for_status').and_raise( module.requests.exceptions.RequestException diff --git a/tests/unit/hooks/monitoring/test_pagerduty.py b/tests/unit/hooks/monitoring/test_pagerduty.py index bf7cc7ba..8da27eb3 100644 --- a/tests/unit/hooks/monitoring/test_pagerduty.py +++ b/tests/unit/hooks/monitoring/test_pagerduty.py @@ -4,6 +4,9 @@ from borgmatic.hooks.monitoring import pagerduty as module def test_ping_monitor_ignores_start_state(): + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').never() module.ping_monitor( @@ -17,6 +20,9 @@ def test_ping_monitor_ignores_start_state(): def test_ping_monitor_ignores_finish_state(): + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').never() module.ping_monitor( @@ -30,6 +36,9 @@ def test_ping_monitor_ignores_finish_state(): def test_ping_monitor_calls_api_for_fail_state(): + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').and_return(flexmock(ok=True)) module.ping_monitor( @@ -43,6 +52,9 @@ def test_ping_monitor_calls_api_for_fail_state(): def test_ping_monitor_dry_run_does_not_call_api(): + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').never() module.ping_monitor( @@ -56,6 +68,9 @@ def test_ping_monitor_dry_run_does_not_call_api(): def test_ping_monitor_with_connection_error_logs_warning(): + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').and_raise( module.requests.exceptions.ConnectionError ) @@ -73,6 +88,9 @@ def test_ping_monitor_with_connection_error_logs_warning(): def test_ping_monitor_with_other_error_logs_warning(): response = flexmock(ok=False) + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) response.should_receive('raise_for_status').and_raise( module.requests.exceptions.RequestException ) diff --git a/tests/unit/hooks/monitoring/test_pushover.py b/tests/unit/hooks/monitoring/test_pushover.py index 281a88f0..ea835ae3 100644 --- a/tests/unit/hooks/monitoring/test_pushover.py +++ b/tests/unit/hooks/monitoring/test_pushover.py @@ -11,6 +11,9 @@ def test_ping_monitor_config_with_minimum_config_fail_state_backup_successfully_ should be auto populated with the default value which is the state name. ''' hook_config = {'token': 'ksdjfwoweijfvwoeifvjmwghagy92', 'user': '983hfe0of902lkjfa2amanfgui'} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -38,6 +41,9 @@ def test_ping_monitor_config_with_minimum_config_start_state_backup_not_send_to_ 'start' state. Only the 'fail' state is enabled by default. ''' hook_config = {'token': 'ksdjfwoweijfvwoeifvjmwghagy92', 'user': '983hfe0of902lkjfa2amanfgui'} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').never() @@ -63,6 +69,9 @@ def test_ping_monitor_start_state_backup_default_message_successfully_send_to_pu 'user': '983hfe0of902lkjfa2amanfgui', 'states': {'start', 'fail', 'finish'}, } + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -96,6 +105,9 @@ def test_ping_monitor_start_state_backup_custom_message_successfully_send_to_pus 'states': {'start', 'fail', 'finish'}, 'start': {'message': 'custom start message'}, } + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -128,6 +140,9 @@ def test_ping_monitor_start_state_backup_default_message_with_priority_emergency 'states': {'start', 'fail', 'finish'}, 'start': {'priority': 2}, } + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -163,6 +178,9 @@ def test_ping_monitor_start_state_backup_default_message_with_priority_emergency 'states': {'start', 'fail', 'finish'}, 'start': {'priority': 2, 'expire': 600}, } + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -198,6 +216,9 @@ def test_ping_monitor_start_state_backup_default_message_with_priority_emergency 'states': {'start', 'fail', 'finish'}, 'start': {'priority': 2, 'retry': 30}, } + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -236,6 +257,9 @@ def test_ping_monitor_start_state_backup_default_message_with_priority_high_decl 'start': {'priority': 1, 'expire': 30, 'retry': 30}, } + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').never() @@ -288,6 +312,9 @@ def test_ping_monitor_start_state_backup_based_on_documentation_advanced_example 'url_title': 'Login to ticketing system', }, } + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -351,6 +378,9 @@ def test_ping_monitor_fail_state_backup_based_on_documentation_advanced_example_ 'url_title': 'Login to ticketing system', }, } + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -419,6 +449,9 @@ def test_ping_monitor_finish_state_backup_based_on_documentation_advanced_exampl 'url_title': 'Login to ticketing system', }, } + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -446,12 +479,15 @@ def test_ping_monitor_finish_state_backup_based_on_documentation_advanced_exampl ) -def test_ping_monitor_config_with_minimum_config_fail_state_backup_successfully_send_to_pushover_dryrun(): +def test_ping_monitor_config_with_minimum_config_fail_state_backup_successfully_send_to_pushover_dry_run(): ''' This test should be the minimum working configuration. The "message" should be auto populated with the default value which is the state name. ''' hook_config = {'token': 'ksdjfwoweijfvwoeifvjmwghagy92', 'user': '983hfe0of902lkjfa2amanfgui'} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').and_return(flexmock(ok=True)).never() @@ -473,6 +509,9 @@ def test_ping_monitor_config_incorrect_state_exit_early(): 'token': 'ksdjfwoweijfvwoeifvjmwghagy92', 'user': '983hfe0of902lkjfa2amanfgui', } + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').and_return(flexmock(ok=True)).never() @@ -486,7 +525,7 @@ def test_ping_monitor_config_incorrect_state_exit_early(): ) -def test_ping_monitor_push_post_error_exits_early(): +def test_ping_monitor_push_post_error_bails(): ''' This test simulates the Pushover servers not responding with a 200 OK. We should raise for status and warn then exit. @@ -496,6 +535,9 @@ def test_ping_monitor_push_post_error_exits_early(): 'user': '983hfe0of902lkjfa2amanfgui', } + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) push_response = flexmock(ok=False) push_response.should_receive('raise_for_status').and_raise( module.requests.ConnectionError diff --git a/tests/unit/hooks/monitoring/test_zabbix.py b/tests/unit/hooks/monitoring/test_zabbix.py index 77193d67..057b4193 100644 --- a/tests/unit/hooks/monitoring/test_zabbix.py +++ b/tests/unit/hooks/monitoring/test_zabbix.py @@ -57,7 +57,7 @@ AUTH_HEADERS_API_KEY = { AUTH_HEADERS_USERNAME_PASSWORD = {'Content-Type': 'application/json-rpc'} -def test_ping_monitor_with_non_matching_state_exits_early(): +def test_ping_monitor_with_non_matching_state_bails(): hook_config = {'api_key': API_KEY} flexmock(module.requests).should_receive('post').never() @@ -71,10 +71,13 @@ def test_ping_monitor_with_non_matching_state_exits_early(): ) -def test_ping_monitor_config_with_api_key_only_exit_early(): +def test_ping_monitor_config_with_api_key_only_bails(): # This test should exit early since only providing an API KEY is not enough # for the hook to work hook_config = {'api_key': API_KEY} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').once() flexmock(module.requests).should_receive('post').never() @@ -88,10 +91,13 @@ def test_ping_monitor_config_with_api_key_only_exit_early(): ) -def test_ping_monitor_config_with_host_only_exit_early(): +def test_ping_monitor_config_with_host_only_bails(): # This test should exit early since only providing a HOST is not enough # for the hook to work hook_config = {'host': HOST} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').once() flexmock(module.requests).should_receive('post').never() @@ -105,10 +111,13 @@ def test_ping_monitor_config_with_host_only_exit_early(): ) -def test_ping_monitor_config_with_key_only_exit_early(): +def test_ping_monitor_config_with_key_only_bails(): # This test should exit early since only providing a KEY is not enough # for the hook to work hook_config = {'key': KEY} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').once() flexmock(module.requests).should_receive('post').never() @@ -122,10 +131,13 @@ def test_ping_monitor_config_with_key_only_exit_early(): ) -def test_ping_monitor_config_with_server_only_exit_early(): +def test_ping_monitor_config_with_server_only_bails(): # This test should exit early since only providing a SERVER is not enough # for the hook to work hook_config = {'server': SERVER} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').once() flexmock(module.requests).should_receive('post').never() @@ -139,9 +151,12 @@ def test_ping_monitor_config_with_server_only_exit_early(): ) -def test_ping_monitor_config_user_password_no_zabbix_data_exit_early(): +def test_ping_monitor_config_user_password_no_zabbix_data_bails(): # This test should exit early since there are HOST/KEY or ITEMID provided to publish data to hook_config = {'server': SERVER, 'username': USERNAME, 'password': PASSWORD} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').once() flexmock(module.requests).should_receive('post').never() @@ -155,9 +170,12 @@ def test_ping_monitor_config_user_password_no_zabbix_data_exit_early(): ) -def test_ping_monitor_config_api_key_no_zabbix_data_exit_early(): +def test_ping_monitor_config_api_key_no_zabbix_data_bails(): # This test should exit early since there are HOST/KEY or ITEMID provided to publish data to hook_config = {'server': SERVER, 'api_key': API_KEY} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').once() flexmock(module.requests).should_receive('post').never() @@ -171,10 +189,13 @@ def test_ping_monitor_config_api_key_no_zabbix_data_exit_early(): ) -def test_ping_monitor_config_itemid_no_auth_data_exit_early(): +def test_ping_monitor_config_itemid_no_auth_data_bails(): # This test should exit early since there is no authentication provided # and Zabbix requires authentication to use it's API hook_config = {'server': SERVER, 'itemid': ITEMID} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').once() flexmock(module.requests).should_receive('post').never() @@ -188,10 +209,13 @@ def test_ping_monitor_config_itemid_no_auth_data_exit_early(): ) -def test_ping_monitor_config_host_and_key_no_auth_data_exit_early(): +def test_ping_monitor_config_host_and_key_no_auth_data_bails(): # This test should exit early since there is no authentication provided # and Zabbix requires authentication to use it's API hook_config = {'server': SERVER, 'host': HOST, 'key': KEY} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').once() flexmock(module.requests).should_receive('post').never() @@ -209,6 +233,9 @@ def test_ping_monitor_config_host_and_key_with_api_key_auth_data_successful(): # This test should simulate a successful POST to a Zabbix server. This test uses API_KEY # to authenticate and HOST/KEY to know which item to populate in Zabbix. hook_config = {'server': SERVER, 'host': HOST, 'key': KEY, 'api_key': API_KEY} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').with_args( f'{SERVER}', headers=AUTH_HEADERS_API_KEY, @@ -226,8 +253,11 @@ def test_ping_monitor_config_host_and_key_with_api_key_auth_data_successful(): ) -def test_ping_monitor_config_host_and_missing_key_exits_early(): +def test_ping_monitor_config_host_and_missing_key_bails(): hook_config = {'server': SERVER, 'host': HOST, 'api_key': API_KEY} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').once() flexmock(module.requests).should_receive('post').never() @@ -241,8 +271,11 @@ def test_ping_monitor_config_host_and_missing_key_exits_early(): ) -def test_ping_monitor_config_key_and_missing_host_exits_early(): +def test_ping_monitor_config_key_and_missing_host_bails(): hook_config = {'server': SERVER, 'key': KEY, 'api_key': API_KEY} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').once() flexmock(module.requests).should_receive('post').never() @@ -267,6 +300,9 @@ def test_ping_monitor_config_host_and_key_with_username_password_auth_data_succe 'password': PASSWORD, } + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) auth_response = flexmock(ok=True) auth_response.should_receive('json').and_return( {'jsonrpc': '2.0', 'result': '3fe6ed01a69ebd79907a120bcd04e494', 'id': 1} @@ -296,7 +332,7 @@ def test_ping_monitor_config_host_and_key_with_username_password_auth_data_succe ) -def test_ping_monitor_config_host_and_key_with_username_password_auth_data_and_auth_post_error_exits_early(): +def test_ping_monitor_config_host_and_key_with_username_password_auth_data_and_auth_post_error_bails(): hook_config = { 'server': SERVER, 'host': HOST, @@ -305,6 +341,9 @@ def test_ping_monitor_config_host_and_key_with_username_password_auth_data_and_a 'password': PASSWORD, } + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) auth_response = flexmock(ok=False) auth_response.should_receive('json').and_return( {'jsonrpc': '2.0', 'result': '3fe6ed01a69ebd79907a120bcd04e494', 'id': 1} @@ -335,7 +374,7 @@ def test_ping_monitor_config_host_and_key_with_username_password_auth_data_and_a ) -def test_ping_monitor_config_host_and_key_with_username_and_missing_password_exits_early(): +def test_ping_monitor_config_host_and_key_with_username_and_missing_password_bails(): hook_config = { 'server': SERVER, 'host': HOST, @@ -343,6 +382,9 @@ def test_ping_monitor_config_host_and_key_with_username_and_missing_password_exi 'username': USERNAME, } + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').once() flexmock(module.requests).should_receive('post').never() @@ -356,7 +398,7 @@ def test_ping_monitor_config_host_and_key_with_username_and_missing_password_exi ) -def test_ping_monitor_config_host_and_key_with_passing_and_missing_username_exits_early(): +def test_ping_monitor_config_host_and_key_with_password_and_missing_username_bails(): hook_config = { 'server': SERVER, 'host': HOST, @@ -364,6 +406,9 @@ def test_ping_monitor_config_host_and_key_with_passing_and_missing_username_exit 'password': PASSWORD, } + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').once() flexmock(module.requests).should_receive('post').never() @@ -381,6 +426,9 @@ def test_ping_monitor_config_itemid_with_api_key_auth_data_successful(): # This test should simulate a successful POST to a Zabbix server. This test uses API_KEY # to authenticate and HOST/KEY to know which item to populate in Zabbix. hook_config = {'server': SERVER, 'itemid': ITEMID, 'api_key': API_KEY} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').with_args( f'{SERVER}', headers=AUTH_HEADERS_API_KEY, @@ -403,6 +451,9 @@ def test_ping_monitor_config_itemid_with_username_password_auth_data_successful( # to authenticate and HOST/KEY to know which item to populate in Zabbix. hook_config = {'server': SERVER, 'itemid': ITEMID, 'username': USERNAME, 'password': PASSWORD} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) auth_response = flexmock(ok=True) auth_response.should_receive('json').and_return( {'jsonrpc': '2.0', 'result': '3fe6ed01a69ebd79907a120bcd04e494', 'id': 1} @@ -432,9 +483,12 @@ def test_ping_monitor_config_itemid_with_username_password_auth_data_successful( ) -def test_ping_monitor_config_itemid_with_username_password_auth_data_and_push_post_error_exits_early(): +def test_ping_monitor_config_itemid_with_username_password_auth_data_and_push_post_error_bails(): hook_config = {'server': SERVER, 'itemid': ITEMID, 'username': USERNAME, 'password': PASSWORD} + flexmock(module.borgmatic.hooks.credential.tag).should_receive( + 'resolve_credential' + ).replace_with(lambda value: value) auth_response = flexmock(ok=True) auth_response.should_receive('json').and_return( {'jsonrpc': '2.0', 'result': '3fe6ed01a69ebd79907a120bcd04e494', 'id': 1} From 73fe29b055021340de0dfb97b75948c6421d45f1 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Feb 2025 09:52:07 -0800 Subject: [PATCH 011/226] Add additional test coverage for credential tag (#966). --- tests/unit/hooks/monitoring/test_ntfy.py | 18 +++++++++++++++ tests/unit/hooks/monitoring/test_pagerduty.py | 17 ++++++++++++++ tests/unit/hooks/monitoring/test_pushover.py | 22 +++++++++++++++++++ tests/unit/hooks/monitoring/test_zabbix.py | 19 ++++++++++++++++ 4 files changed, 76 insertions(+) diff --git a/tests/unit/hooks/monitoring/test_ntfy.py b/tests/unit/hooks/monitoring/test_ntfy.py index efbecf96..d6317071 100644 --- a/tests/unit/hooks/monitoring/test_ntfy.py +++ b/tests/unit/hooks/monitoring/test_ntfy.py @@ -309,6 +309,24 @@ def test_ping_monitor_with_connection_error_logs_warning(): ) +def test_ping_monitor_with_credential_error_logs_warning(): + hook_config = {'topic': topic} + flexmock(module.borgmatic.hooks.credential.tag).should_receive('resolve_credential').and_raise( + ValueError + ) + flexmock(module.requests).should_receive('post').never() + flexmock(module.logger).should_receive('warning').once() + + module.ping_monitor( + hook_config, + {}, + 'config.yaml', + borgmatic.hooks.monitoring.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, + ) + + def test_ping_monitor_with_other_error_logs_warning(): hook_config = {'topic': topic} flexmock(module.borgmatic.hooks.credential.tag).should_receive( diff --git a/tests/unit/hooks/monitoring/test_pagerduty.py b/tests/unit/hooks/monitoring/test_pagerduty.py index 8da27eb3..115387f7 100644 --- a/tests/unit/hooks/monitoring/test_pagerduty.py +++ b/tests/unit/hooks/monitoring/test_pagerduty.py @@ -86,6 +86,23 @@ def test_ping_monitor_with_connection_error_logs_warning(): ) +def test_ping_monitor_with_credential_error_logs_warning(): + flexmock(module.borgmatic.hooks.credential.tag).should_receive('resolve_credential').and_raise( + ValueError + ) + flexmock(module.requests).should_receive('post').never() + flexmock(module.logger).should_receive('warning') + + module.ping_monitor( + {'integration_key': 'abc123'}, + {}, + 'config.yaml', + module.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, + ) + + def test_ping_monitor_with_other_error_logs_warning(): response = flexmock(ok=False) flexmock(module.borgmatic.hooks.credential.tag).should_receive( diff --git a/tests/unit/hooks/monitoring/test_pushover.py b/tests/unit/hooks/monitoring/test_pushover.py index ea835ae3..17354b7b 100644 --- a/tests/unit/hooks/monitoring/test_pushover.py +++ b/tests/unit/hooks/monitoring/test_pushover.py @@ -562,3 +562,25 @@ def test_ping_monitor_push_post_error_bails(): monitoring_log_level=1, dry_run=False, ) + + +def test_ping_monitor_credential_error_bails(): + hook_config = hook_config = { + 'token': 'ksdjfwoweijfvwoeifvjmwghagy92', + 'user': '983hfe0of902lkjfa2amanfgui', + } + + flexmock(module.borgmatic.hooks.credential.tag).should_receive('resolve_credential').and_raise( + ValueError + ) + flexmock(module.requests).should_receive('post').never() + flexmock(module.logger).should_receive('warning').once() + + module.ping_monitor( + hook_config, + {}, + 'config.yaml', + borgmatic.hooks.monitoring.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, + ) diff --git a/tests/unit/hooks/monitoring/test_zabbix.py b/tests/unit/hooks/monitoring/test_zabbix.py index 057b4193..3429af51 100644 --- a/tests/unit/hooks/monitoring/test_zabbix.py +++ b/tests/unit/hooks/monitoring/test_zabbix.py @@ -520,3 +520,22 @@ def test_ping_monitor_config_itemid_with_username_password_auth_data_and_push_po monitoring_log_level=1, dry_run=False, ) + + +def test_ping_monitor_with_credential_error_bails(): + hook_config = {'server': SERVER, 'itemid': ITEMID, 'username': USERNAME, 'password': PASSWORD} + + flexmock(module.borgmatic.hooks.credential.tag).should_receive('resolve_credential').and_raise( + ValueError + ) + flexmock(module.requests).should_receive('post').never() + flexmock(module.logger).should_receive('warning').once() + + module.ping_monitor( + hook_config, + {}, + 'config.yaml', + borgmatic.hooks.monitoring.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, + ) From 24b846e9ca1e373cd8c2ce60a577e5ba600186a4 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Feb 2025 10:05:51 -0800 Subject: [PATCH 012/226] Additional test coverage (#966). --- tests/unit/borg/test_environment.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/unit/borg/test_environment.py b/tests/unit/borg/test_environment.py index fda097b7..ff10104c 100644 --- a/tests/unit/borg/test_environment.py +++ b/tests/unit/borg/test_environment.py @@ -33,6 +33,21 @@ def test_make_environment_with_passphrase_should_set_environment(): assert environment.get('BORG_PASSPHRASE') == 'pass' +def test_make_environment_with_credential_tag_passphrase_should_load_it_and_set_environment(): + flexmock(module.borgmatic.borg.passcommand).should_receive( + 'get_passphrase_from_passcommand' + ).and_return(None) + flexmock(module.os).should_receive('pipe').never() + flexmock(module.os.environ).should_receive('get').and_return(None) + flexmock(module.borgmatic.hooks.credential.tag).should_receive('resolve_credential').with_args( + '!credential systemd pass' + ).and_return('pass') + + environment = module.make_environment({'encryption_passphrase': '!credential systemd pass'}) + + assert environment.get('BORG_PASSPHRASE') == 'pass' + + def test_make_environment_with_ssh_command_should_set_environment(): flexmock(module.borgmatic.borg.passcommand).should_receive( 'get_passphrase_from_passcommand' From 779f51f40a84fd7d610057327ace46ab4d8a4991 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Feb 2025 13:24:27 -0800 Subject: [PATCH 013/226] Fix favicon on non-home pages. --- docs/_includes/layouts/base.njk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_includes/layouts/base.njk b/docs/_includes/layouts/base.njk index 1119e27b..0807f5f0 100644 --- a/docs/_includes/layouts/base.njk +++ b/docs/_includes/layouts/base.njk @@ -4,7 +4,7 @@ - + {{ subtitle + ' - ' if subtitle}}{{ title }} {%- set css %} {% include 'index.css' %} From 3bc14ba364021be83df4734a335c4a7bdb36b34e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Feb 2025 14:21:33 -0800 Subject: [PATCH 014/226] Bump version for release. --- NEWS | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index a86324e6..ed3038b1 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -1.9.10.dev0 +1.9.10 * #966: Add a "!credential" tag for loading systemd credentials into borgmatic configuration files. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/ diff --git a/pyproject.toml b/pyproject.toml index 92102e3f..e47d0762 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "borgmatic" -version = "1.9.10.dev0" +version = "1.9.10" authors = [ { name="Dan Helfman", email="witten@torsion.org" }, ] From 50096296daf016de3ac37485b0982e890697a045 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Feb 2025 22:01:23 -0800 Subject: [PATCH 015/226] Revamp systemd credential syntax to be more consistent with constants (#966). --- NEWS | 4 +- borgmatic/borg/environment.py | 4 +- borgmatic/config/load.py | 16 ----- borgmatic/config/schema.yaml | 64 ++++++++++--------- borgmatic/hooks/credential/parse.py | 42 ++++++++++++ borgmatic/hooks/credential/tag.py | 27 -------- borgmatic/hooks/data_source/mariadb.py | 12 ++-- borgmatic/hooks/data_source/mongodb.py | 10 +-- borgmatic/hooks/data_source/mysql.py | 12 ++-- borgmatic/hooks/data_source/postgresql.py | 17 +++-- borgmatic/hooks/monitoring/ntfy.py | 8 +-- borgmatic/hooks/monitoring/pagerduty.py | 4 +- borgmatic/hooks/monitoring/pushover.py | 6 +- borgmatic/hooks/monitoring/zabbix.py | 8 +-- docs/how-to/provide-your-passwords.md | 22 +++---- .../hooks/credential/test_systemd.py | 2 +- tests/integration/config/test_load.py | 14 ---- tests/unit/borg/test_environment.py | 10 +-- tests/unit/config/test_load.py | 14 ---- .../credential/{test_tag.py => test_parse.py} | 15 +++-- tests/unit/hooks/data_source/test_mariadb.py | 44 ++++++------- tests/unit/hooks/data_source/test_mongodb.py | 24 +++---- tests/unit/hooks/data_source/test_mysql.py | 40 ++++++------ .../unit/hooks/data_source/test_postgresql.py | 60 ++++++++--------- tests/unit/hooks/monitoring/test_ntfy.py | 34 +++++----- tests/unit/hooks/monitoring/test_pagerduty.py | 18 +++--- tests/unit/hooks/monitoring/test_pushover.py | 34 +++++----- tests/unit/hooks/monitoring/test_zabbix.py | 42 ++++++------ 28 files changed, 295 insertions(+), 312 deletions(-) create mode 100644 borgmatic/hooks/credential/parse.py delete mode 100644 borgmatic/hooks/credential/tag.py rename tests/unit/hooks/credential/{test_tag.py => test_parse.py} (67%) diff --git a/NEWS b/NEWS index ed3038b1..d9eeea57 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,6 @@ 1.9.10 - * #966: Add a "!credential" tag for loading systemd credentials into borgmatic configuration - files. See the documentation for more information: + * #966: Add a "{credential ...}" syntax for loading systemd credentials into borgmatic + configuration files. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/ * #987: Fix a "list" action error when the "encryption_passcommand" option is set. * #987: When both "encryption_passcommand" and "encryption_passphrase" are configured, prefer diff --git a/borgmatic/borg/environment.py b/borgmatic/borg/environment.py index 85033309..5411623d 100644 --- a/borgmatic/borg/environment.py +++ b/borgmatic/borg/environment.py @@ -1,7 +1,7 @@ import os import borgmatic.borg.passcommand -import borgmatic.hooks.credential.tag +import borgmatic.hooks.credential.parse OPTION_TO_ENVIRONMENT_VARIABLE = { 'borg_base_directory': 'BORG_BASE_DIR', @@ -41,7 +41,7 @@ def make_environment(config): value = config.get(option_name) if option_name in CREDENTIAL_OPTIONS and value is not None: - value = borgmatic.hooks.credential.tag.resolve_credential(value) + value = borgmatic.hooks.credential.parse.resolve_credential(value) if value is not None: environment[environment_variable_name] = str(value) diff --git a/borgmatic/config/load.py b/borgmatic/config/load.py index 7a4d75ee..864bf3e5 100644 --- a/borgmatic/config/load.py +++ b/borgmatic/config/load.py @@ -104,21 +104,6 @@ def raise_omit_node_error(loader, node): ) -def reserialize_tag_node(loader, tag_node): - ''' - Given a ruamel.yaml loader and a node for a tag and value, convert the node back into a string - of the form "!tagname value" and return it. The idea is that downstream code, rather than this - file's YAML loading logic, should be responsible for interpreting this particular tag—since the - downstream code actually understands the meaning behind the tag. - - Raise ValueError if the tag node's value isn't a string. - ''' - if isinstance(tag_node.value, str): - return f'{tag_node.tag} {tag_node.value}' - - raise ValueError(f'The value given for the {tag_node.tag} tag is invalid; use a string instead') - - class Include_constructor(ruamel.yaml.SafeConstructor): ''' A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including @@ -137,7 +122,6 @@ class Include_constructor(ruamel.yaml.SafeConstructor): config_paths=config_paths, ), ) - self.add_constructor('!credential', reserialize_tag_node) # These are catch-all error handlers for tags that don't get applied and removed by # deep_merge_nodes() below. diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 7c8e5b0a..bc59edbc 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -250,7 +250,7 @@ properties: repositories that were initialized with passphrase/repokey/keyfile encryption. Quote the value if it contains punctuation, so it parses correctly. And backslash any quote or backslash literals as well. - Defaults to not set. Supports the "!credential" tag. + Defaults to not set. Supports the "{credential ...}" syntax. example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" checkpoint_interval: type: integer @@ -989,13 +989,15 @@ properties: Username with which to connect to the database. Defaults to the username of the current user. You probably want to specify the "postgres" superuser here when the - database name is "all". Supports the "!credential" tag. + database name is "all". Supports the "{credential ...}" + syntax. example: dbuser restore_username: type: string description: | Username with which to restore the database. Defaults to - the "username" option. Supports the "!credential" tag. + the "username" option. Supports the "{credential ...}" + syntax. example: dbuser password: type: string @@ -1003,14 +1005,15 @@ properties: Password with which to connect to the database. Omitting a password will only work if PostgreSQL is configured to trust the configured username without a password or you - create a ~/.pgpass file. Supports the "!credential" tag. + create a ~/.pgpass file. Supports the "{credential ...}" + syntax. example: trustsome1 restore_password: type: string description: | Password with which to connect to the restore database. Defaults to the "password" option. Supports the - "!credential" tag. + "{credential ...}" syntax. example: trustsome1 no_owner: type: boolean @@ -1171,13 +1174,14 @@ properties: description: | Username with which to connect to the database. Defaults to the username of the current user. Supports the - "!credential" tag. + "{credential ...}" syntax. example: dbuser restore_username: type: string description: | Username with which to restore the database. Defaults to - the "username" option. Supports the "!credential" tag. + the "username" option. Supports the "{credential ...}" + syntax. example: dbuser password: type: string @@ -1185,14 +1189,14 @@ properties: Password with which to connect to the database. Omitting a password will only work if MariaDB is configured to trust the configured username without a password. - Supports the "!credential" tag. + Supports the "{credential ...}" syntax. example: trustsome1 restore_password: type: string description: | Password with which to connect to the restore database. Defaults to the "password" option. Supports the - "!credential" tag. + "{credential ...}" syntax. example: trustsome1 mariadb_dump_command: type: string @@ -1300,13 +1304,14 @@ properties: description: | Username with which to connect to the database. Defaults to the username of the current user. Supports the - "!credential" tag. + "{credential ...}" syntax. example: dbuser restore_username: type: string description: | Username with which to restore the database. Defaults to - the "username" option. Supports the "!credential" tag. + the "username" option. Supports the "{credential ...}" + syntax. example: dbuser password: type: string @@ -1314,14 +1319,14 @@ properties: Password with which to connect to the database. Omitting a password will only work if MySQL is configured to trust the configured username without a password. - Supports the "!credential" tag. + Supports the "{credential ...}" syntax. example: trustsome1 restore_password: type: string description: | Password with which to connect to the restore database. Defaults to the "password" option. Supports the - "!credential" tag. + "{credential ...}" syntax. example: trustsome1 mysql_dump_command: type: string @@ -1459,27 +1464,28 @@ properties: description: | Username with which to connect to the database. Skip it if no authentication is needed. Supports the - "!credential" tag. + "{credential ...}" syntax. example: dbuser restore_username: type: string description: | Username with which to restore the database. Defaults to - the "username" option. Supports the "!credential" tag. + the "username" option. Supports the "{credential ...}" + syntax. example: dbuser password: type: string description: | Password with which to connect to the database. Skip it if no authentication is needed. Supports the - "!credential" tag. + "{credential ...}" syntax. example: trustsome1 restore_password: type: string description: | Password with which to connect to the restore database. Defaults to the "password" option. Supports the - "!credential" tag. + "{credential ...}" syntax. example: trustsome1 authentication_database: type: string @@ -1539,19 +1545,19 @@ properties: type: string description: | The username used for authentication. Supports the - "!credential" tag. + "{credential ...}" syntax. example: testuser password: type: string description: | The password used for authentication. Supports the - "!credential" tag. + "{credential ...}" syntax. example: fakepassword access_token: type: string description: | An ntfy access token to authenticate with instead of - username/password. Supports the "!credential" tag. + username/password. Supports the "{credential ...}" syntax. example: tk_AgQdq7mVBoFD37zQVN29RhuMzNIz2 start: type: object @@ -1646,8 +1652,8 @@ properties: token: type: string description: | - Your application's API token. Supports the "!credential" - tag. + Your application's API token. Supports the "{credential + ...}" syntax. example: 7ms6TXHpTokTou2P6x4SodDeentHRa user: type: string @@ -1655,7 +1661,7 @@ properties: Your user/group key (or that of your target user), viewable when logged into your dashboard: often referred to as USER_KEY in Pushover documentation and code examples. - Supports the "!credential" tag. + Supports the "{credential ...}" syntax. example: hwRwoWsXMBWwgrSecfa9EfPey55WSN start: type: object @@ -1929,19 +1935,19 @@ properties: type: string description: | The username used for authentication. Not needed if using - an API key. Supports the "!credential" tag. + an API key. Supports the "{credential ...}" syntax. example: testuser password: type: string description: | The password used for authentication. Not needed if using - an API key. Supports the "!credential" tag. + an API key. Supports the "{credential ...}" syntax. example: fakepassword api_key: type: string description: | - The API key used for authentication. Not needed if using - an username/password. Supports the "!credential" tag. + The API key used for authentication. Not needed if using an + username/password. Supports the "{credential ...}" syntax. example: fakekey start: type: object @@ -2221,8 +2227,8 @@ properties: integration_key: type: string description: | - PagerDuty integration key used to notify PagerDuty - when a backup errors. Supports the "!credential" tag. + PagerDuty integration key used to notify PagerDuty when a + backup errors. Supports the "{credential ...}" syntax. example: a177cad45bd374409f78906a810a3074 description: | Configuration for a monitoring integration with PagerDuty. Create an diff --git a/borgmatic/hooks/credential/parse.py b/borgmatic/hooks/credential/parse.py new file mode 100644 index 00000000..dff0196f --- /dev/null +++ b/borgmatic/hooks/credential/parse.py @@ -0,0 +1,42 @@ +import functools +import re + +import borgmatic.hooks.dispatch + +IS_A_HOOK = False + + +CREDENTIAL_PATTERN = re.compile( + r'\{credential +(?P[A-Za-z0-9_]+) +(?P[A-Za-z0-9_]+)\}' +) + +GENERAL_CREDENTIAL_PATTERN = re.compile(r'\{credential( +[^}]*)?\}') + + +@functools.cache +def resolve_credential(value): + ''' + Given a configuration value containing a string like "{credential hookname credentialname}", resolve it by + calling the relevant hook to get the actual credential value. If the given value does not + actually contain a credential tag, then return it unchanged. + + Cache the value so repeated calls to this function don't need to load the credential repeatedly. + + Raise ValueError if the config could not be parsed or the credential could not be loaded. + ''' + if value is None: + return value + + result = CREDENTIAL_PATTERN.sub( + lambda matcher: borgmatic.hooks.dispatch.call_hook( + 'load_credential', {}, matcher.group('hook_name'), matcher.group('credential_name') + ), + value, + ) + + # If we've tried to parse the credential, but the parsed result still looks kind of like a + # credential, it means it's invalid syntax. + if GENERAL_CREDENTIAL_PATTERN.match(result): + raise ValueError(f'Cannot load credential with invalid syntax "{value}"') + + return result diff --git a/borgmatic/hooks/credential/tag.py b/borgmatic/hooks/credential/tag.py deleted file mode 100644 index 634ca84d..00000000 --- a/borgmatic/hooks/credential/tag.py +++ /dev/null @@ -1,27 +0,0 @@ -import functools - -import borgmatic.hooks.dispatch - -IS_A_HOOK = False - - -@functools.cache -def resolve_credential(tag): - ''' - Given a configuration tag string like "!credential hookname credentialname", resolve it by - calling the relevant hook to get the actual credential value. If the given tag is not actually a - credential tag, then return the value unchanged. - - Cache the value so repeated calls to this function don't need to load the credential repeatedly. - - Raise ValueError if the config could not be parsed or the credential could not be loaded. - ''' - if tag and tag.startswith('!credential '): - try: - (tag_name, hook_name, credential_name) = tag.split(' ', 2) - except ValueError: - raise ValueError(f'Cannot load credential with invalid syntax "{tag}"') - - return borgmatic.hooks.dispatch.call_hook('load_credential', {}, hook_name, credential_name) - - return tag diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index dcdd431f..11e3eddc 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -5,7 +5,7 @@ import shlex import borgmatic.borg.pattern import borgmatic.config.paths -import borgmatic.hooks.credential.tag +import borgmatic.hooks.credential.parse from borgmatic.execute import ( execute_command, execute_command_and_capture_output, @@ -47,7 +47,7 @@ def database_names_to_dump(database, extra_environment, dry_run): + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + ( - ('--user', borgmatic.hooks.credential.tag.resolve_credential(database['username'])) + ('--user', borgmatic.hooks.credential.parse.resolve_credential(database['username'])) if 'username' in database else () ) @@ -102,7 +102,7 @@ def execute_dump_command( + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + ( - ('--user', borgmatic.hooks.credential.tag.resolve_credential(database['username'])) + ('--user', borgmatic.hooks.credential.parse.resolve_credential(database['username'])) if 'username' in database else () ) @@ -162,7 +162,7 @@ def dump_data_sources( for database in databases: dump_path = make_dump_path(borgmatic_runtime_directory) extra_environment = ( - {'MYSQL_PWD': borgmatic.hooks.credential.tag.resolve_credential(database['password'])} + {'MYSQL_PWD': borgmatic.hooks.credential.parse.resolve_credential(database['password'])} if 'password' in database else None ) @@ -264,11 +264,11 @@ def restore_data_source_dump( port = str( connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')) ) - username = borgmatic.hooks.credential.tag.resolve_credential( + username = borgmatic.hooks.credential.parse.resolve_credential( connection_params['username'] or data_source.get('restore_username', data_source.get('username')) ) - password = borgmatic.hooks.credential.tag.resolve_credential( + password = borgmatic.hooks.credential.parse.resolve_credential( connection_params['password'] or data_source.get('restore_password', data_source.get('password')) ) diff --git a/borgmatic/hooks/data_source/mongodb.py b/borgmatic/hooks/data_source/mongodb.py index 55dc6e9c..8015d35f 100644 --- a/borgmatic/hooks/data_source/mongodb.py +++ b/borgmatic/hooks/data_source/mongodb.py @@ -4,7 +4,7 @@ import shlex import borgmatic.borg.pattern import borgmatic.config.paths -import borgmatic.hooks.credential.tag +import borgmatic.hooks.credential.parse from borgmatic.execute import execute_command, execute_command_with_processes from borgmatic.hooks.data_source import dump @@ -103,7 +103,7 @@ def build_dump_command(database, dump_filename, dump_format): ( '--username', shlex.quote( - borgmatic.hooks.credential.tag.resolve_credential(database['username']) + borgmatic.hooks.credential.parse.resolve_credential(database['username']) ), ) if 'username' in database @@ -113,7 +113,7 @@ def build_dump_command(database, dump_filename, dump_format): ( '--password', shlex.quote( - borgmatic.hooks.credential.tag.resolve_credential(database['password']) + borgmatic.hooks.credential.parse.resolve_credential(database['password']) ), ) if 'password' in database @@ -217,10 +217,10 @@ def build_restore_command(extract_process, database, dump_filename, connection_p 'restore_hostname', database.get('hostname') ) port = str(connection_params['port'] or database.get('restore_port', database.get('port', ''))) - username = borgmatic.hooks.credential.tag.resolve_credential( + username = borgmatic.hooks.credential.parse.resolve_credential( connection_params['username'] or database.get('restore_username', database.get('username')) ) - password = borgmatic.hooks.credential.tag.resolve_credential( + password = borgmatic.hooks.credential.parse.resolve_credential( connection_params['password'] or database.get('restore_password', database.get('password')) ) diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index 5be8b71e..4b9fd320 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -5,7 +5,7 @@ import shlex import borgmatic.borg.pattern import borgmatic.config.paths -import borgmatic.hooks.credential.tag +import borgmatic.hooks.credential.parse from borgmatic.execute import ( execute_command, execute_command_and_capture_output, @@ -47,7 +47,7 @@ def database_names_to_dump(database, extra_environment, dry_run): + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + ( - ('--user', borgmatic.hooks.credential.tag.resolve_credential(database['username'])) + ('--user', borgmatic.hooks.credential.parse.resolve_credential(database['username'])) if 'username' in database else () ) @@ -101,7 +101,7 @@ def execute_dump_command( + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + ( - ('--user', borgmatic.hooks.credential.tag.resolve_credential(database['username'])) + ('--user', borgmatic.hooks.credential.parse.resolve_credential(database['username'])) if 'username' in database else () ) @@ -161,7 +161,7 @@ def dump_data_sources( for database in databases: dump_path = make_dump_path(borgmatic_runtime_directory) extra_environment = ( - {'MYSQL_PWD': borgmatic.hooks.credential.tag.resolve_credential(database['password'])} + {'MYSQL_PWD': borgmatic.hooks.credential.parse.resolve_credential(database['password'])} if 'password' in database else None ) @@ -263,11 +263,11 @@ def restore_data_source_dump( port = str( connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')) ) - username = borgmatic.hooks.credential.tag.resolve_credential( + username = borgmatic.hooks.credential.parse.resolve_credential( connection_params['username'] or data_source.get('restore_username', data_source.get('username')) ) - password = borgmatic.hooks.credential.tag.resolve_credential( + password = borgmatic.hooks.credential.parse.resolve_credential( connection_params['password'] or data_source.get('restore_password', data_source.get('password')) ) diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index cfacf867..82678286 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -7,7 +7,7 @@ import shlex import borgmatic.borg.pattern import borgmatic.config.paths -import borgmatic.hooks.credential.tag +import borgmatic.hooks.credential.parse from borgmatic.execute import ( execute_command, execute_command_and_capture_output, @@ -34,12 +34,12 @@ def make_extra_environment(database, restore_connection_params=None): try: if restore_connection_params: - extra['PGPASSWORD'] = borgmatic.hooks.credential.tag.resolve_credential( + extra['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential( restore_connection_params.get('password') or database.get('restore_password', database['password']) ) else: - extra['PGPASSWORD'] = borgmatic.hooks.credential.tag.resolve_credential( + extra['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential( database['password'] ) except (AttributeError, KeyError): @@ -87,7 +87,10 @@ def database_names_to_dump(database, extra_environment, dry_run): + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + ( - ('--username', borgmatic.hooks.credential.tag.resolve_credential(database['username'])) + ( + '--username', + borgmatic.hooks.credential.parse.resolve_credential(database['username']), + ) if 'username' in database else () ) @@ -185,7 +188,9 @@ def dump_data_sources( ( '--username', shlex.quote( - borgmatic.hooks.credential.tag.resolve_credential(database['username']) + borgmatic.hooks.credential.parse.resolve_credential( + database['username'] + ) ), ) if 'username' in database @@ -303,7 +308,7 @@ def restore_data_source_dump( port = str( connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')) ) - username = borgmatic.hooks.credential.tag.resolve_credential( + username = borgmatic.hooks.credential.parse.resolve_credential( connection_params['username'] or data_source.get('restore_username', data_source.get('username')) ) diff --git a/borgmatic/hooks/monitoring/ntfy.py b/borgmatic/hooks/monitoring/ntfy.py index d0bf1dee..fb4adc14 100644 --- a/borgmatic/hooks/monitoring/ntfy.py +++ b/borgmatic/hooks/monitoring/ntfy.py @@ -2,7 +2,7 @@ import logging import requests -import borgmatic.hooks.credential.tag +import borgmatic.hooks.credential.parse logger = logging.getLogger(__name__) @@ -50,13 +50,13 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev } try: - username = borgmatic.hooks.credential.tag.resolve_credential( + username = borgmatic.hooks.credential.parse.resolve_credential( hook_config.get('username') ) - password = borgmatic.hooks.credential.tag.resolve_credential( + password = borgmatic.hooks.credential.parse.resolve_credential( hook_config.get('password') ) - access_token = borgmatic.hooks.credential.tag.resolve_credential( + access_token = borgmatic.hooks.credential.parse.resolve_credential( hook_config.get('access_token') ) except ValueError as error: diff --git a/borgmatic/hooks/monitoring/pagerduty.py b/borgmatic/hooks/monitoring/pagerduty.py index 53839eff..d7a54e62 100644 --- a/borgmatic/hooks/monitoring/pagerduty.py +++ b/borgmatic/hooks/monitoring/pagerduty.py @@ -5,7 +5,7 @@ import platform import requests -import borgmatic.hooks.credential.tag +import borgmatic.hooks.credential.parse from borgmatic.hooks.monitoring import monitor logger = logging.getLogger(__name__) @@ -41,7 +41,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev return try: - integration_key = borgmatic.hooks.credential.tag.resolve_credential( + integration_key = borgmatic.hooks.credential.parse.resolve_credential( hook_config.get('integration_key') ) except ValueError as error: diff --git a/borgmatic/hooks/monitoring/pushover.py b/borgmatic/hooks/monitoring/pushover.py index 4f86ee0d..61afdb98 100644 --- a/borgmatic/hooks/monitoring/pushover.py +++ b/borgmatic/hooks/monitoring/pushover.py @@ -2,7 +2,7 @@ import logging import requests -import borgmatic.hooks.credential.tag +import borgmatic.hooks.credential.parse logger = logging.getLogger(__name__) @@ -35,8 +35,8 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev state_config = hook_config.get(state.name.lower(), {}) try: - token = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('token')) - user = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('user')) + token = borgmatic.hooks.credential.parse.resolve_credential(hook_config.get('token')) + user = borgmatic.hooks.credential.parse.resolve_credential(hook_config.get('user')) except ValueError as error: logger.warning(f'Pushover credential error: {error}') return diff --git a/borgmatic/hooks/monitoring/zabbix.py b/borgmatic/hooks/monitoring/zabbix.py index f66c93a5..df8bfed9 100644 --- a/borgmatic/hooks/monitoring/zabbix.py +++ b/borgmatic/hooks/monitoring/zabbix.py @@ -2,7 +2,7 @@ import logging import requests -import borgmatic.hooks.credential.tag +import borgmatic.hooks.credential.parse logger = logging.getLogger(__name__) @@ -37,9 +37,9 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev ) try: - username = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('username')) - password = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('password')) - api_key = borgmatic.hooks.credential.tag.resolve_credential(hook_config.get('api_key')) + username = borgmatic.hooks.credential.parse.resolve_credential(hook_config.get('username')) + password = borgmatic.hooks.credential.parse.resolve_credential(hook_config.get('password')) + api_key = borgmatic.hooks.credential.parse.resolve_credential(hook_config.get('api_key')) except ValueError as error: logger.warning(f'Zabbix credential error: {error}') return diff --git a/docs/how-to/provide-your-passwords.md b/docs/how-to/provide-your-passwords.md index 6e908c78..01bd6a9c 100644 --- a/docs/how-to/provide-your-passwords.md +++ b/docs/how-to/provide-your-passwords.md @@ -62,7 +62,7 @@ systemd-ask-password -n | systemd-creds encrypt - /etc/credstore.encrypted/borgm Then use the following in your configuration file: ```yaml -encryption_passphrase: !credential systemd borgmatic.pw +encryption_passphrase: "{credential systemd borgmatic.pw}" ``` Prior to version 1.9.10 You can @@ -74,16 +74,16 @@ encryption_passcommand: cat ${CREDENTIALS_DIRECTORY}/borgmatic.pw Note that the name `borgmatic.pw` is hardcoded in the systemd service file. -The `!credential` tag works for several different options in a borgmatic +The `{credential ...}` syntax works for several different options in a borgmatic configuration file besides just `encryption_passphrase`. For instance, the username, password, and API token options within database and monitoring hooks -support `!credential`. For example: +support `{credential ...}`: ```yaml postgresql_databases: - name: invoices username: postgres - password: !credential systemd borgmatic_db1 + password: "{credential systemd borgmatic_db1}" ``` For specifics about which options are supported, see the @@ -121,7 +121,7 @@ Finally, use something like the following in your borgmatic configuration file for each option value you'd like to load from systemd: ```yaml -encryption_passphrase: !credential systemd borgmatic_backupserver1 +encryption_passphrase: "{credential systemd borgmatic_backupserver1}" ``` Prior to version 1.9.10 Use the @@ -135,12 +135,12 @@ encryption_passcommand: cat ${CREDENTIALS_DIRECTORY}/borgmatic_backupserver1 Adjust `borgmatic_backupserver1` according to the name of the credential and the directory set in the service file. -Be aware that when using this systemd `!credential` feature, you may no longer -be able to run certain borgmatic actions outside of the systemd service, as the -credentials are only available from within the context of that service. So for -instance, `borgmatic list` necessarily relies on the `encryption_passphrase` in -order to access the Borg repository, but it shouldn't need to load any -credentials for your database or monitoring hooks. +Be aware that when using this systemd `{credential ...}` feature, you may no +longer be able to run certain borgmatic actions outside of the systemd service, +as the credentials are only available from within the context of that service. +So for instance, `borgmatic list` necessarily relies on the +`encryption_passphrase` in order to access the Borg repository, but `list` +shouldn't need to load any credentials for your database or monitoring hooks. The one exception is `borgmatic config validate`, which doesn't actually load any credentials and should continue working anywhere. diff --git a/tests/end-to-end/hooks/credential/test_systemd.py b/tests/end-to-end/hooks/credential/test_systemd.py index b4d43966..d58302c3 100644 --- a/tests/end-to-end/hooks/credential/test_systemd.py +++ b/tests/end-to-end/hooks/credential/test_systemd.py @@ -23,7 +23,7 @@ def generate_configuration(config_path, repository_path): .replace('- /home', f'- {config_path}') .replace('- /etc', '') .replace('- /var/log/syslog*', '') - + '\nencryption_passphrase: !credential systemd mycredential' + + '\nencryption_passphrase: "{credential systemd mycredential}"' ) config_file = open(config_path, 'w') config_file.write(config) diff --git a/tests/integration/config/test_load.py b/tests/integration/config/test_load.py index ab57db64..e13c69c8 100644 --- a/tests/integration/config/test_load.py +++ b/tests/integration/config/test_load.py @@ -225,20 +225,6 @@ def test_load_configuration_merges_multiple_file_include(): assert config_paths == {'config.yaml', '/tmp/include1.yaml', '/tmp/include2.yaml', 'other.yaml'} -def test_load_configuration_passes_through_credential_tag(): - builtins = flexmock(sys.modules['builtins']) - flexmock(module.os).should_receive('getcwd').and_return('/tmp') - flexmock(module.os.path).should_receive('isabs').and_return(False) - flexmock(module.os.path).should_receive('exists').and_return(True) - config_file = io.StringIO('key: !credential foo bar') - config_file.name = 'config.yaml' - builtins.should_receive('open').with_args('config.yaml').and_return(config_file) - config_paths = {'other.yaml'} - - assert module.load_configuration('config.yaml', config_paths) == {'key': '!credential foo bar'} - assert config_paths == {'config.yaml', 'other.yaml'} - - def test_load_configuration_with_retain_tag_merges_include_but_keeps_local_values(): builtins = flexmock(sys.modules['builtins']) flexmock(module.os).should_receive('getcwd').and_return('/tmp') diff --git a/tests/unit/borg/test_environment.py b/tests/unit/borg/test_environment.py index ff10104c..f118d3b1 100644 --- a/tests/unit/borg/test_environment.py +++ b/tests/unit/borg/test_environment.py @@ -24,7 +24,7 @@ def test_make_environment_with_passphrase_should_set_environment(): ).and_return(None) flexmock(module.os).should_receive('pipe').never() flexmock(module.os.environ).should_receive('get').and_return(None) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) @@ -39,11 +39,11 @@ def test_make_environment_with_credential_tag_passphrase_should_load_it_and_set_ ).and_return(None) flexmock(module.os).should_receive('pipe').never() flexmock(module.os.environ).should_receive('get').and_return(None) - flexmock(module.borgmatic.hooks.credential.tag).should_receive('resolve_credential').with_args( - '!credential systemd pass' - ).and_return('pass') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).with_args('{credential systemd pass}').and_return('pass') - environment = module.make_environment({'encryption_passphrase': '!credential systemd pass'}) + environment = module.make_environment({'encryption_passphrase': '{credential systemd pass}'}) assert environment.get('BORG_PASSPHRASE') == 'pass' diff --git a/tests/unit/config/test_load.py b/tests/unit/config/test_load.py index e1fe2c52..4c1d221c 100644 --- a/tests/unit/config/test_load.py +++ b/tests/unit/config/test_load.py @@ -43,17 +43,3 @@ def test_probe_and_include_file_with_relative_path_and_missing_files_raises(): with pytest.raises(FileNotFoundError): module.probe_and_include_file('include.yaml', ['/etc', '/var'], config_paths=set()) - - -def test_reserialize_tag_node_turns_it_into_string(): - assert ( - module.reserialize_tag_node(loader=flexmock(), tag_node=flexmock(tag='!tag', value='value')) - == '!tag value' - ) - - -def test_reserialize_tag_node_with_invalid_value_raises(): - with pytest.raises(ValueError): - assert module.reserialize_tag_node( - loader=flexmock(), tag_node=flexmock(tag='!tag', value=['value']) - ) diff --git a/tests/unit/hooks/credential/test_tag.py b/tests/unit/hooks/credential/test_parse.py similarity index 67% rename from tests/unit/hooks/credential/test_tag.py rename to tests/unit/hooks/credential/test_parse.py index d802ce69..9c2ca8e3 100644 --- a/tests/unit/hooks/credential/test_tag.py +++ b/tests/unit/hooks/credential/test_parse.py @@ -1,14 +1,14 @@ import pytest from flexmock import flexmock -from borgmatic.hooks.credential import tag as module +from borgmatic.hooks.credential import parse as module def test_resolve_credential_passes_through_string_without_credential_tag(): module.resolve_credential.cache_clear() flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').never() - assert module.resolve_credential('!no credentials here') == '!no credentials here' + assert module.resolve_credential('{no credentials here}') == '{no credentials here}' def test_resolve_credential_passes_through_none(): @@ -18,12 +18,13 @@ def test_resolve_credential_passes_through_none(): assert module.resolve_credential(None) is None -def test_resolve_credential_with_invalid_credential_tag_raises(): +@pytest.mark.parametrize('invalid_value', ('{credential}', '{credential }', '{credential systemd}')) +def test_resolve_credential_with_invalid_credential_tag_raises(invalid_value): module.resolve_credential.cache_clear() flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').never() with pytest.raises(ValueError): - module.resolve_credential('!credential systemd') + module.resolve_credential(invalid_value) def test_resolve_credential_with_valid_credential_tag_loads_credential(): @@ -35,7 +36,7 @@ def test_resolve_credential_with_valid_credential_tag_loads_credential(): 'mycredential', ).and_return('result').once() - assert module.resolve_credential('!credential systemd mycredential') == 'result' + assert module.resolve_credential('{credential systemd mycredential}') == 'result' def test_resolve_credential_caches_credential_after_first_call(): @@ -47,5 +48,5 @@ def test_resolve_credential_caches_credential_after_first_call(): 'mycredential', ).and_return('result').once() - assert module.resolve_credential('!credential systemd mycredential') == 'result' - assert module.resolve_credential('!credential systemd mycredential') == 'result' + assert module.resolve_credential('{credential systemd mycredential}') == 'result' + assert module.resolve_credential('{credential systemd mycredential}') == 'result' diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py index f354e8d8..e55af230 100644 --- a/tests/unit/hooks/data_source/test_mariadb.py +++ b/tests/unit/hooks/data_source/test_mariadb.py @@ -25,7 +25,7 @@ def test_database_names_to_dump_bails_for_dry_run(): def test_database_names_to_dump_queries_mariadb_for_database_names(): extra_environment = flexmock() - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -53,7 +53,7 @@ def test_dump_data_sources_dumps_each_database(): databases = [{'name': 'foo'}, {'name': 'bar'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( @@ -87,7 +87,7 @@ def test_dump_data_sources_dumps_with_password(): database = {'name': 'foo', 'username': 'root', 'password': 'trustsome1'} process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( @@ -117,7 +117,7 @@ def test_dump_data_sources_dumps_all_databases_at_once(): databases = [{'name': 'all'}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) @@ -144,7 +144,7 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured databases = [{'name': 'all', 'format': 'sql'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) @@ -214,7 +214,7 @@ def test_execute_dump_command_runs_mariadb_dump(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') @@ -249,7 +249,7 @@ def test_execute_dump_command_runs_mariadb_dump_without_add_drop_database(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') @@ -283,7 +283,7 @@ def test_execute_dump_command_runs_mariadb_dump_with_hostname_and_port(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') @@ -324,7 +324,7 @@ def test_execute_dump_command_runs_mariadb_dump_with_username_and_password(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') @@ -361,7 +361,7 @@ def test_execute_dump_command_runs_mariadb_dump_with_options(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') @@ -397,7 +397,7 @@ def test_execute_dump_command_runs_non_default_mariadb_dump_with_options(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') @@ -455,7 +455,7 @@ def test_execute_dump_command_with_duplicate_dump_skips_mariadb_dump(): def test_execute_dump_command_with_dry_run_skips_mariadb_dump(): flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') @@ -478,7 +478,7 @@ def test_execute_dump_command_with_dry_run_skips_mariadb_dump(): def test_dump_data_sources_errors_for_missing_all_databases(): databases = [{'name': 'all'}] flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( @@ -500,7 +500,7 @@ def test_dump_data_sources_errors_for_missing_all_databases(): def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run(): databases = [{'name': 'all'}] flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( @@ -525,7 +525,7 @@ def test_restore_data_source_dump_runs_mariadb_to_restore(): hook_config = [{'name': 'foo'}, {'name': 'bar'}] extract_process = flexmock(stdout=flexmock()) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -556,7 +556,7 @@ def test_restore_data_source_dump_runs_mariadb_with_options(): hook_config = [{'name': 'foo', 'restore_options': '--harder'}] extract_process = flexmock(stdout=flexmock()) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -589,7 +589,7 @@ def test_restore_data_source_dump_runs_non_default_mariadb_with_options(): ] extract_process = flexmock(stdout=flexmock()) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -620,7 +620,7 @@ def test_restore_data_source_dump_runs_mariadb_with_hostname_and_port(): hook_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] extract_process = flexmock(stdout=flexmock()) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -660,7 +660,7 @@ def test_restore_data_source_dump_runs_mariadb_with_username_and_password(): hook_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}] extract_process = flexmock(stdout=flexmock()) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -701,7 +701,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ ] extract_process = flexmock(stdout=flexmock()) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -755,7 +755,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ ] extract_process = flexmock(stdout=flexmock()) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -796,7 +796,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ def test_restore_data_source_dump_with_dry_run_skips_restore(): hook_config = [{'name': 'foo'}] - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').never() diff --git a/tests/unit/hooks/data_source/test_mongodb.py b/tests/unit/hooks/data_source/test_mongodb.py index 1bc17e9a..4603bfa9 100644 --- a/tests/unit/hooks/data_source/test_mongodb.py +++ b/tests/unit/hooks/data_source/test_mongodb.py @@ -124,7 +124,7 @@ def test_dump_data_sources_runs_mongodump_with_username_and_password(): flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( 'databases/localhost/foo' ) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') @@ -245,7 +245,7 @@ def test_dump_data_sources_runs_mongodumpall_for_all_databases(): def test_build_dump_command_with_username_injection_attack_gets_escaped(): database = {'name': 'test', 'username': 'bob; naughty-command'} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) @@ -260,7 +260,7 @@ def test_restore_data_source_dump_runs_mongorestore(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -294,7 +294,7 @@ def test_restore_data_source_dump_runs_mongorestore_with_hostname_and_port(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -342,7 +342,7 @@ def test_restore_data_source_dump_runs_mongorestore_with_username_and_password() flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -396,7 +396,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -454,7 +454,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -500,7 +500,7 @@ def test_restore_data_source_dump_runs_mongorestore_with_options(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -532,7 +532,7 @@ def test_restore_databases_dump_runs_mongorestore_with_schemas(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -572,7 +572,7 @@ def test_restore_data_source_dump_runs_psql_for_all_database_dump(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -603,7 +603,7 @@ def test_restore_data_source_dump_with_dry_run_skips_restore(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').never() @@ -629,7 +629,7 @@ def test_restore_data_source_dump_without_extract_process_restores_from_disk(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('/dump/path') - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( diff --git a/tests/unit/hooks/data_source/test_mysql.py b/tests/unit/hooks/data_source/test_mysql.py index e81182c7..252b23a5 100644 --- a/tests/unit/hooks/data_source/test_mysql.py +++ b/tests/unit/hooks/data_source/test_mysql.py @@ -16,7 +16,7 @@ def test_database_names_to_dump_passes_through_name(): def test_database_names_to_dump_bails_for_dry_run(): extra_environment = flexmock() - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_and_capture_output').never() @@ -28,7 +28,7 @@ def test_database_names_to_dump_bails_for_dry_run(): def test_database_names_to_dump_queries_mysql_for_database_names(): extra_environment = flexmock() - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -87,7 +87,7 @@ def test_dump_data_sources_dumps_with_password(): database = {'name': 'foo', 'username': 'root', 'password': 'trustsome1'} process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( @@ -208,7 +208,7 @@ def test_execute_dump_command_runs_mysqldump(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') @@ -243,7 +243,7 @@ def test_execute_dump_command_runs_mysqldump_without_add_drop_database(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') @@ -277,7 +277,7 @@ def test_execute_dump_command_runs_mysqldump_with_hostname_and_port(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') @@ -318,7 +318,7 @@ def test_execute_dump_command_runs_mysqldump_with_username_and_password(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') @@ -355,7 +355,7 @@ def test_execute_dump_command_runs_mysqldump_with_options(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') @@ -391,7 +391,7 @@ def test_execute_dump_command_runs_non_default_mysqldump(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') @@ -447,7 +447,7 @@ def test_execute_dump_command_with_duplicate_dump_skips_mysqldump(): def test_execute_dump_command_with_dry_run_skips_mysqldump(): flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') @@ -470,7 +470,7 @@ def test_execute_dump_command_with_dry_run_skips_mysqldump(): def test_dump_data_sources_errors_for_missing_all_databases(): databases = [{'name': 'all'}] flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( @@ -492,7 +492,7 @@ def test_dump_data_sources_errors_for_missing_all_databases(): def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run(): databases = [{'name': 'all'}] flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( @@ -517,7 +517,7 @@ def test_restore_data_source_dump_runs_mysql_to_restore(): hook_config = [{'name': 'foo'}, {'name': 'bar'}] extract_process = flexmock(stdout=flexmock()) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -548,7 +548,7 @@ def test_restore_data_source_dump_runs_mysql_with_options(): hook_config = [{'name': 'foo', 'restore_options': '--harder'}] extract_process = flexmock(stdout=flexmock()) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -579,7 +579,7 @@ def test_restore_data_source_dump_runs_non_default_mysql_with_options(): hook_config = [{'name': 'foo', 'mysql_command': 'custom_mysql', 'restore_options': '--harder'}] extract_process = flexmock(stdout=flexmock()) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -610,7 +610,7 @@ def test_restore_data_source_dump_runs_mysql_with_hostname_and_port(): hook_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] extract_process = flexmock(stdout=flexmock()) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -650,7 +650,7 @@ def test_restore_data_source_dump_runs_mysql_with_username_and_password(): hook_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}] extract_process = flexmock(stdout=flexmock()) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -691,7 +691,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ ] extract_process = flexmock(stdout=flexmock()) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -745,7 +745,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ ] extract_process = flexmock(stdout=flexmock()) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -786,7 +786,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ def test_restore_data_source_dump_with_dry_run_skips_restore(): hook_config = [{'name': 'foo'}] - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_with_processes').never() diff --git a/tests/unit/hooks/data_source/test_postgresql.py b/tests/unit/hooks/data_source/test_postgresql.py index 05f3f7dd..3abd0740 100644 --- a/tests/unit/hooks/data_source/test_postgresql.py +++ b/tests/unit/hooks/data_source/test_postgresql.py @@ -24,7 +24,7 @@ def test_make_extra_environment_maps_options_to_environment(): 'PGSSLROOTCERT': 'root.crt', 'PGSSLCRL': 'crl.crl', } - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) @@ -35,7 +35,7 @@ def test_make_extra_environment_maps_options_to_environment(): def test_make_extra_environment_with_cli_password_sets_correct_password(): database = {'name': 'foo', 'restore_password': 'trustsome1', 'password': 'anotherpassword'} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) @@ -84,7 +84,7 @@ def test_database_names_to_dump_passes_through_all_without_format(): def test_database_names_to_dump_with_all_and_format_and_dry_run_bails(): database = {'name': 'all', 'format': 'custom'} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_and_capture_output').never() @@ -94,7 +94,7 @@ def test_database_names_to_dump_with_all_and_format_and_dry_run_bails(): def test_database_names_to_dump_with_all_and_format_lists_databases(): database = {'name': 'all', 'format': 'custom'} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_and_capture_output').and_return( @@ -109,7 +109,7 @@ def test_database_names_to_dump_with_all_and_format_lists_databases(): def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostname_and_port(): database = {'name': 'all', 'format': 'custom', 'hostname': 'localhost', 'port': 1234} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -136,7 +136,7 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostnam def test_database_names_to_dump_with_all_and_format_lists_databases_with_username(): database = {'name': 'all', 'format': 'custom', 'username': 'postgres'} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -161,7 +161,7 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_usernam def test_database_names_to_dump_with_all_and_format_lists_databases_with_options(): database = {'name': 'all', 'format': 'custom', 'list_options': '--harder'} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -177,7 +177,7 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_options def test_database_names_to_dump_with_all_and_format_excludes_particular_databases(): database = {'name': 'all', 'format': 'custom'} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_and_capture_output').and_return( @@ -193,7 +193,7 @@ def test_database_names_to_dump_with_all_and_psql_command_uses_custom_command(): 'format': 'custom', 'psql_command': 'docker exec --workdir * mycontainer psql', } - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('execute_command_and_capture_output').with_args( @@ -246,7 +246,7 @@ def test_dump_data_sources_runs_pg_dump_for_each_database(): 'databases/localhost/foo' ).and_return('databases/localhost/bar') flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') @@ -353,7 +353,7 @@ def test_dump_data_sources_with_dry_run_skips_pg_dump(): 'databases/localhost/foo' ).and_return('databases/localhost/bar') flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() @@ -382,7 +382,7 @@ def test_dump_data_sources_runs_pg_dump_with_hostname_and_port(): 'databases/database.example.org/foo' ) flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') @@ -430,7 +430,7 @@ def test_dump_data_sources_runs_pg_dump_with_username_and_password(): 'databases/localhost/foo' ) flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') @@ -476,7 +476,7 @@ def test_dump_data_sources_with_username_injection_attack_gets_escaped(): 'databases/localhost/foo' ) flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') @@ -519,7 +519,7 @@ def test_dump_data_sources_runs_pg_dump_with_directory_format(): 'databases/localhost/foo' ) flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_parent_directory_for_dump') @@ -564,7 +564,7 @@ def test_dump_data_sources_runs_pg_dump_with_options(): 'databases/localhost/foo' ) flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') @@ -607,7 +607,7 @@ def test_dump_data_sources_runs_pg_dumpall_for_all_databases(): 'databases/localhost/all' ) flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') @@ -639,7 +639,7 @@ def test_dump_data_sources_runs_non_default_pg_dump(): 'databases/localhost/foo' ) flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') @@ -677,7 +677,7 @@ def test_restore_data_source_dump_runs_pg_restore(): hook_config = [{'name': 'foo', 'schemas': None}, {'name': 'bar'}] extract_process = flexmock(stdout=flexmock()) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) @@ -734,7 +734,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_hostname_and_port(): ] extract_process = flexmock(stdout=flexmock()) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) @@ -799,7 +799,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_username_and_password(): ] extract_process = flexmock(stdout=flexmock()) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('make_extra_environment').and_return( @@ -873,7 +873,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ ] extract_process = flexmock(stdout=flexmock()) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('make_extra_environment').and_return( @@ -955,7 +955,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ ] extract_process = flexmock(stdout=flexmock()) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('make_extra_environment').and_return( @@ -1031,7 +1031,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_options(): ] extract_process = flexmock(stdout=flexmock()) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) @@ -1088,7 +1088,7 @@ def test_restore_data_source_dump_runs_psql_for_all_database_dump(): hook_config = [{'name': 'all', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) @@ -1130,7 +1130,7 @@ def test_restore_data_source_dump_runs_psql_for_plain_database_dump(): hook_config = [{'name': 'foo', 'format': 'plain', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) @@ -1184,7 +1184,7 @@ def test_restore_data_source_dump_runs_non_default_pg_restore_and_psql(): ] extract_process = flexmock(stdout=flexmock()) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) @@ -1248,7 +1248,7 @@ def test_restore_data_source_dump_runs_non_default_pg_restore_and_psql(): def test_restore_data_source_dump_with_dry_run_skips_restore(): hook_config = [{'name': 'foo', 'schemas': None}] - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) @@ -1275,7 +1275,7 @@ def test_restore_data_source_dump_with_dry_run_skips_restore(): def test_restore_data_source_dump_without_extract_process_restores_from_disk(): hook_config = [{'name': 'foo', 'schemas': None}] - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) @@ -1330,7 +1330,7 @@ def test_restore_data_source_dump_without_extract_process_restores_from_disk(): def test_restore_data_source_dump_with_schemas_restores_schemas(): hook_config = [{'name': 'foo', 'schemas': ['bar', 'baz']}] - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) diff --git a/tests/unit/hooks/monitoring/test_ntfy.py b/tests/unit/hooks/monitoring/test_ntfy.py index d6317071..3d8e21d4 100644 --- a/tests/unit/hooks/monitoring/test_ntfy.py +++ b/tests/unit/hooks/monitoring/test_ntfy.py @@ -36,7 +36,7 @@ def return_default_message_headers(state=Enum): def test_ping_monitor_minimal_config_hits_hosted_ntfy_on_fail(): hook_config = {'topic': topic} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').with_args( @@ -60,7 +60,7 @@ def test_ping_monitor_with_access_token_hits_hosted_ntfy_on_fail(): 'topic': topic, 'access_token': 'abc123', } - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').with_args( @@ -86,7 +86,7 @@ def test_ping_monitor_with_username_password_and_access_token_ignores_username_p 'password': 'fakepassword', 'access_token': 'abc123', } - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').with_args( @@ -112,7 +112,7 @@ def test_ping_monitor_with_username_password_hits_hosted_ntfy_on_fail(): 'username': 'testuser', 'password': 'fakepassword', } - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').with_args( @@ -133,7 +133,7 @@ def test_ping_monitor_with_username_password_hits_hosted_ntfy_on_fail(): def test_ping_monitor_with_password_but_no_username_warns(): hook_config = {'topic': topic, 'password': 'fakepassword'} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').with_args( @@ -155,7 +155,7 @@ def test_ping_monitor_with_password_but_no_username_warns(): def test_ping_monitor_with_username_but_no_password_warns(): hook_config = {'topic': topic, 'username': 'testuser'} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').with_args( @@ -177,7 +177,7 @@ def test_ping_monitor_with_username_but_no_password_warns(): def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_start(): hook_config = {'topic': topic} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').never() @@ -194,7 +194,7 @@ def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_start(): def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_finish(): hook_config = {'topic': topic} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').never() @@ -211,7 +211,7 @@ def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_finish(): def test_ping_monitor_minimal_config_hits_selfhosted_ntfy_on_fail(): hook_config = {'topic': topic, 'server': custom_base_url} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').with_args( @@ -232,7 +232,7 @@ def test_ping_monitor_minimal_config_hits_selfhosted_ntfy_on_fail(): def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_fail_dry_run(): hook_config = {'topic': topic} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').never() @@ -249,7 +249,7 @@ def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_fail_dry_run(): def test_ping_monitor_custom_message_hits_hosted_ntfy_on_fail(): hook_config = {'topic': topic, 'fail': custom_message_config} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').with_args( @@ -268,7 +268,7 @@ def test_ping_monitor_custom_message_hits_hosted_ntfy_on_fail(): def test_ping_monitor_custom_state_hits_hosted_ntfy_on_start(): hook_config = {'topic': topic, 'states': ['start', 'fail']} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').with_args( @@ -289,7 +289,7 @@ def test_ping_monitor_custom_state_hits_hosted_ntfy_on_start(): def test_ping_monitor_with_connection_error_logs_warning(): hook_config = {'topic': topic} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').with_args( @@ -311,9 +311,9 @@ def test_ping_monitor_with_connection_error_logs_warning(): def test_ping_monitor_with_credential_error_logs_warning(): hook_config = {'topic': topic} - flexmock(module.borgmatic.hooks.credential.tag).should_receive('resolve_credential').and_raise( - ValueError - ) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).and_raise(ValueError) flexmock(module.requests).should_receive('post').never() flexmock(module.logger).should_receive('warning').once() @@ -329,7 +329,7 @@ def test_ping_monitor_with_credential_error_logs_warning(): def test_ping_monitor_with_other_error_logs_warning(): hook_config = {'topic': topic} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) response = flexmock(ok=False) diff --git a/tests/unit/hooks/monitoring/test_pagerduty.py b/tests/unit/hooks/monitoring/test_pagerduty.py index 115387f7..aff03f76 100644 --- a/tests/unit/hooks/monitoring/test_pagerduty.py +++ b/tests/unit/hooks/monitoring/test_pagerduty.py @@ -4,7 +4,7 @@ from borgmatic.hooks.monitoring import pagerduty as module def test_ping_monitor_ignores_start_state(): - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').never() @@ -20,7 +20,7 @@ def test_ping_monitor_ignores_start_state(): def test_ping_monitor_ignores_finish_state(): - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').never() @@ -36,7 +36,7 @@ def test_ping_monitor_ignores_finish_state(): def test_ping_monitor_calls_api_for_fail_state(): - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').and_return(flexmock(ok=True)) @@ -52,7 +52,7 @@ def test_ping_monitor_calls_api_for_fail_state(): def test_ping_monitor_dry_run_does_not_call_api(): - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').never() @@ -68,7 +68,7 @@ def test_ping_monitor_dry_run_does_not_call_api(): def test_ping_monitor_with_connection_error_logs_warning(): - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').and_raise( @@ -87,9 +87,9 @@ def test_ping_monitor_with_connection_error_logs_warning(): def test_ping_monitor_with_credential_error_logs_warning(): - flexmock(module.borgmatic.hooks.credential.tag).should_receive('resolve_credential').and_raise( - ValueError - ) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).and_raise(ValueError) flexmock(module.requests).should_receive('post').never() flexmock(module.logger).should_receive('warning') @@ -105,7 +105,7 @@ def test_ping_monitor_with_credential_error_logs_warning(): def test_ping_monitor_with_other_error_logs_warning(): response = flexmock(ok=False) - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) response.should_receive('raise_for_status').and_raise( diff --git a/tests/unit/hooks/monitoring/test_pushover.py b/tests/unit/hooks/monitoring/test_pushover.py index 17354b7b..b10e5e14 100644 --- a/tests/unit/hooks/monitoring/test_pushover.py +++ b/tests/unit/hooks/monitoring/test_pushover.py @@ -11,7 +11,7 @@ def test_ping_monitor_config_with_minimum_config_fail_state_backup_successfully_ should be auto populated with the default value which is the state name. ''' hook_config = {'token': 'ksdjfwoweijfvwoeifvjmwghagy92', 'user': '983hfe0of902lkjfa2amanfgui'} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() @@ -41,7 +41,7 @@ def test_ping_monitor_config_with_minimum_config_start_state_backup_not_send_to_ 'start' state. Only the 'fail' state is enabled by default. ''' hook_config = {'token': 'ksdjfwoweijfvwoeifvjmwghagy92', 'user': '983hfe0of902lkjfa2amanfgui'} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() @@ -69,7 +69,7 @@ def test_ping_monitor_start_state_backup_default_message_successfully_send_to_pu 'user': '983hfe0of902lkjfa2amanfgui', 'states': {'start', 'fail', 'finish'}, } - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() @@ -105,7 +105,7 @@ def test_ping_monitor_start_state_backup_custom_message_successfully_send_to_pus 'states': {'start', 'fail', 'finish'}, 'start': {'message': 'custom start message'}, } - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() @@ -140,7 +140,7 @@ def test_ping_monitor_start_state_backup_default_message_with_priority_emergency 'states': {'start', 'fail', 'finish'}, 'start': {'priority': 2}, } - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() @@ -178,7 +178,7 @@ def test_ping_monitor_start_state_backup_default_message_with_priority_emergency 'states': {'start', 'fail', 'finish'}, 'start': {'priority': 2, 'expire': 600}, } - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() @@ -216,7 +216,7 @@ def test_ping_monitor_start_state_backup_default_message_with_priority_emergency 'states': {'start', 'fail', 'finish'}, 'start': {'priority': 2, 'retry': 30}, } - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() @@ -257,7 +257,7 @@ def test_ping_monitor_start_state_backup_default_message_with_priority_high_decl 'start': {'priority': 1, 'expire': 30, 'retry': 30}, } - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() @@ -312,7 +312,7 @@ def test_ping_monitor_start_state_backup_based_on_documentation_advanced_example 'url_title': 'Login to ticketing system', }, } - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() @@ -378,7 +378,7 @@ def test_ping_monitor_fail_state_backup_based_on_documentation_advanced_example_ 'url_title': 'Login to ticketing system', }, } - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() @@ -449,7 +449,7 @@ def test_ping_monitor_finish_state_backup_based_on_documentation_advanced_exampl 'url_title': 'Login to ticketing system', }, } - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() @@ -485,7 +485,7 @@ def test_ping_monitor_config_with_minimum_config_fail_state_backup_successfully_ should be auto populated with the default value which is the state name. ''' hook_config = {'token': 'ksdjfwoweijfvwoeifvjmwghagy92', 'user': '983hfe0of902lkjfa2amanfgui'} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() @@ -509,7 +509,7 @@ def test_ping_monitor_config_incorrect_state_exit_early(): 'token': 'ksdjfwoweijfvwoeifvjmwghagy92', 'user': '983hfe0of902lkjfa2amanfgui', } - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').never() @@ -535,7 +535,7 @@ def test_ping_monitor_push_post_error_bails(): 'user': '983hfe0of902lkjfa2amanfgui', } - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) push_response = flexmock(ok=False) @@ -570,9 +570,9 @@ def test_ping_monitor_credential_error_bails(): 'user': '983hfe0of902lkjfa2amanfgui', } - flexmock(module.borgmatic.hooks.credential.tag).should_receive('resolve_credential').and_raise( - ValueError - ) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).and_raise(ValueError) flexmock(module.requests).should_receive('post').never() flexmock(module.logger).should_receive('warning').once() diff --git a/tests/unit/hooks/monitoring/test_zabbix.py b/tests/unit/hooks/monitoring/test_zabbix.py index 3429af51..cba05ada 100644 --- a/tests/unit/hooks/monitoring/test_zabbix.py +++ b/tests/unit/hooks/monitoring/test_zabbix.py @@ -75,7 +75,7 @@ def test_ping_monitor_config_with_api_key_only_bails(): # This test should exit early since only providing an API KEY is not enough # for the hook to work hook_config = {'api_key': API_KEY} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').once() @@ -95,7 +95,7 @@ def test_ping_monitor_config_with_host_only_bails(): # This test should exit early since only providing a HOST is not enough # for the hook to work hook_config = {'host': HOST} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').once() @@ -115,7 +115,7 @@ def test_ping_monitor_config_with_key_only_bails(): # This test should exit early since only providing a KEY is not enough # for the hook to work hook_config = {'key': KEY} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').once() @@ -135,7 +135,7 @@ def test_ping_monitor_config_with_server_only_bails(): # This test should exit early since only providing a SERVER is not enough # for the hook to work hook_config = {'server': SERVER} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').once() @@ -154,7 +154,7 @@ def test_ping_monitor_config_with_server_only_bails(): def test_ping_monitor_config_user_password_no_zabbix_data_bails(): # This test should exit early since there are HOST/KEY or ITEMID provided to publish data to hook_config = {'server': SERVER, 'username': USERNAME, 'password': PASSWORD} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').once() @@ -173,7 +173,7 @@ def test_ping_monitor_config_user_password_no_zabbix_data_bails(): def test_ping_monitor_config_api_key_no_zabbix_data_bails(): # This test should exit early since there are HOST/KEY or ITEMID provided to publish data to hook_config = {'server': SERVER, 'api_key': API_KEY} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').once() @@ -193,7 +193,7 @@ def test_ping_monitor_config_itemid_no_auth_data_bails(): # This test should exit early since there is no authentication provided # and Zabbix requires authentication to use it's API hook_config = {'server': SERVER, 'itemid': ITEMID} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').once() @@ -213,7 +213,7 @@ def test_ping_monitor_config_host_and_key_no_auth_data_bails(): # This test should exit early since there is no authentication provided # and Zabbix requires authentication to use it's API hook_config = {'server': SERVER, 'host': HOST, 'key': KEY} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').once() @@ -233,7 +233,7 @@ def test_ping_monitor_config_host_and_key_with_api_key_auth_data_successful(): # This test should simulate a successful POST to a Zabbix server. This test uses API_KEY # to authenticate and HOST/KEY to know which item to populate in Zabbix. hook_config = {'server': SERVER, 'host': HOST, 'key': KEY, 'api_key': API_KEY} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').with_args( @@ -255,7 +255,7 @@ def test_ping_monitor_config_host_and_key_with_api_key_auth_data_successful(): def test_ping_monitor_config_host_and_missing_key_bails(): hook_config = {'server': SERVER, 'host': HOST, 'api_key': API_KEY} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').once() @@ -273,7 +273,7 @@ def test_ping_monitor_config_host_and_missing_key_bails(): def test_ping_monitor_config_key_and_missing_host_bails(): hook_config = {'server': SERVER, 'key': KEY, 'api_key': API_KEY} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').once() @@ -300,7 +300,7 @@ def test_ping_monitor_config_host_and_key_with_username_password_auth_data_succe 'password': PASSWORD, } - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) auth_response = flexmock(ok=True) @@ -341,7 +341,7 @@ def test_ping_monitor_config_host_and_key_with_username_password_auth_data_and_a 'password': PASSWORD, } - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) auth_response = flexmock(ok=False) @@ -382,7 +382,7 @@ def test_ping_monitor_config_host_and_key_with_username_and_missing_password_bai 'username': USERNAME, } - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').once() @@ -406,7 +406,7 @@ def test_ping_monitor_config_host_and_key_with_password_and_missing_username_bai 'password': PASSWORD, } - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.logger).should_receive('warning').once() @@ -426,7 +426,7 @@ def test_ping_monitor_config_itemid_with_api_key_auth_data_successful(): # This test should simulate a successful POST to a Zabbix server. This test uses API_KEY # to authenticate and HOST/KEY to know which item to populate in Zabbix. hook_config = {'server': SERVER, 'itemid': ITEMID, 'api_key': API_KEY} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) flexmock(module.requests).should_receive('post').with_args( @@ -451,7 +451,7 @@ def test_ping_monitor_config_itemid_with_username_password_auth_data_successful( # to authenticate and HOST/KEY to know which item to populate in Zabbix. hook_config = {'server': SERVER, 'itemid': ITEMID, 'username': USERNAME, 'password': PASSWORD} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) auth_response = flexmock(ok=True) @@ -486,7 +486,7 @@ def test_ping_monitor_config_itemid_with_username_password_auth_data_successful( def test_ping_monitor_config_itemid_with_username_password_auth_data_and_push_post_error_bails(): hook_config = {'server': SERVER, 'itemid': ITEMID, 'username': USERNAME, 'password': PASSWORD} - flexmock(module.borgmatic.hooks.credential.tag).should_receive( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value: value) auth_response = flexmock(ok=True) @@ -525,9 +525,9 @@ def test_ping_monitor_config_itemid_with_username_password_auth_data_and_push_po def test_ping_monitor_with_credential_error_bails(): hook_config = {'server': SERVER, 'itemid': ITEMID, 'username': USERNAME, 'password': PASSWORD} - flexmock(module.borgmatic.hooks.credential.tag).should_receive('resolve_credential').and_raise( - ValueError - ) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).and_raise(ValueError) flexmock(module.requests).should_receive('post').never() flexmock(module.logger).should_receive('warning').once() From a0ba5b673b9e4f554b85235cb30b7f5cfc39398e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 11 Feb 2025 22:54:07 -0800 Subject: [PATCH 016/226] Add credential loading from file, KeePassXC, and Docker/Podman secrets. --- NEWS | 3 +++ borgmatic/hooks/credential/container.py | 35 ++++++++++++++++++++++++ borgmatic/hooks/credential/file.py | 29 ++++++++++++++++++++ borgmatic/hooks/credential/keepassxc.py | 36 +++++++++++++++++++++++++ borgmatic/hooks/credential/parse.py | 30 ++++++++++++--------- borgmatic/hooks/credential/systemd.py | 30 ++++++++++----------- pyproject.toml | 2 +- 7 files changed, 136 insertions(+), 29 deletions(-) create mode 100644 borgmatic/hooks/credential/container.py create mode 100644 borgmatic/hooks/credential/file.py create mode 100644 borgmatic/hooks/credential/keepassxc.py diff --git a/NEWS b/NEWS index d9eeea57..3f3e5b78 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +1.9.11 + * Add credential loading from file, KeePassXC, and Docker/Podman secrets. See the documentation for + more information: https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/ 1.9.10 * #966: Add a "{credential ...}" syntax for loading systemd credentials into borgmatic configuration files. See the documentation for more information: diff --git a/borgmatic/hooks/credential/container.py b/borgmatic/hooks/credential/container.py new file mode 100644 index 00000000..2bf976be --- /dev/null +++ b/borgmatic/hooks/credential/container.py @@ -0,0 +1,35 @@ +import logging +import os +import re + +logger = logging.getLogger(__name__) + + +SECRET_NAME_PATTERN = re.compile(r'^\w+$') +SECRETS_DIRECTORY = '/run/secrets' + + +def load_credential(hook_config, config, credential_parameters): + ''' + Given the hook configuration dict, the configuration dict, and a credential parameters tuple + containing a secret name to load, read the secret from the corresponding container secrets file + and return it. + + Raise ValueError if the credential parameters is not one element, the secret name is invalid, or + the secret file cannot be read. + ''' + try: + (secret_name,) = credential_parameters + except ValueError: + raise ValueError(f'Cannot load invalid secret name: "{' '.join(credential_parameters)}"') + + if not SECRET_NAME_PATTERN.match(secret_name): + raise ValueError(f'Cannot load invalid secret name: "{secret_name}"') + + try: + with open(os.path.join(SECRETS_DIRECTORY, secret_name)) as secret_file: + return secret_file.read().rstrip(os.linesep) + except (FileNotFoundError, OSError) as error: + logger.warning(error) + + raise ValueError(f'Cannot load secret "{secret_name}" from file: {error.filename}') diff --git a/borgmatic/hooks/credential/file.py b/borgmatic/hooks/credential/file.py new file mode 100644 index 00000000..e50ff199 --- /dev/null +++ b/borgmatic/hooks/credential/file.py @@ -0,0 +1,29 @@ +import logging +import os +import shlex + +import borgmatic.execute + +logger = logging.getLogger(__name__) + + +def load_credential(hook_config, config, credential_parameters): + ''' + Given the hook configuration dict, the configuration dict, and a credential parameters tuple + containing a credential path to load, load the credential from file and return it. + + Raise ValueError if the credential parameters is not one element or the secret file cannot be + read. + ''' + try: + (credential_path,) = credential_parameters + except ValueError: + raise ValueError(f'Cannot load credential with invalid credential path: "{' '.join(credential_parameters)}"') + + try: + with open(credential_path) as credential_file: + return credential_file.read().rstrip(os.linesep) + except (FileNotFoundError, OSError) as error: + logger.warning(error) + + raise ValueError(f'Cannot load credential "{credential_name}" from file: {error.filename}') diff --git a/borgmatic/hooks/credential/keepassxc.py b/borgmatic/hooks/credential/keepassxc.py new file mode 100644 index 00000000..5543a0ce --- /dev/null +++ b/borgmatic/hooks/credential/keepassxc.py @@ -0,0 +1,36 @@ +import logging +import os +import shlex + +import borgmatic.execute + +logger = logging.getLogger(__name__) + + +def load_credential(hook_config, config, credential_parameters): + ''' + Given the hook configuration dict, the configuration dict, and a credential parameters tuple + containing a KeePassXC database path and an attribute name to load, run keepassxc-cli to fetch + the corresponidng KeePassXC credential and return it. + + Raise ValueError if keepassxc-cli can't retrieve the credential. + ''' + try: + (database_path, attribute_name) = credential_parameters + except ValueError: + raise ValueError(f'Cannot load credential with invalid KeePassXC database path and attribute name: "{' '.join(credential_parameters)}"') + + if not os.path.exists(database_path): + raise ValueError(f'Cannot load credential because KeePassXC database path does not exist: {database_path}') + + return borgmatic.execute.execute_command_and_capture_output( + ( + 'keepassxc-cli', + 'show', + '--show-protected', + '--attributes', + 'Password', + database_path, + attribute_name, + ) + ).rstrip(os.linesep) diff --git a/borgmatic/hooks/credential/parse.py b/borgmatic/hooks/credential/parse.py index dff0196f..9125d235 100644 --- a/borgmatic/hooks/credential/parse.py +++ b/borgmatic/hooks/credential/parse.py @@ -1,5 +1,6 @@ import functools import re +import shlex import borgmatic.hooks.dispatch @@ -7,11 +8,9 @@ IS_A_HOOK = False CREDENTIAL_PATTERN = re.compile( - r'\{credential +(?P[A-Za-z0-9_]+) +(?P[A-Za-z0-9_]+)\}' + r'\{credential( +(?P.*))?\}' ) -GENERAL_CREDENTIAL_PATTERN = re.compile(r'\{credential( +[^}]*)?\}') - @functools.cache def resolve_credential(value): @@ -27,16 +26,21 @@ def resolve_credential(value): if value is None: return value - result = CREDENTIAL_PATTERN.sub( - lambda matcher: borgmatic.hooks.dispatch.call_hook( - 'load_credential', {}, matcher.group('hook_name'), matcher.group('credential_name') - ), - value, - ) + matcher = CREDENTIAL_PATTERN.match(value) - # If we've tried to parse the credential, but the parsed result still looks kind of like a - # credential, it means it's invalid syntax. - if GENERAL_CREDENTIAL_PATTERN.match(result): + if not matcher: + return value + + contents = matcher.group('contents') + + if not contents: raise ValueError(f'Cannot load credential with invalid syntax "{value}"') - return result + (hook_name, *credential_parameters) = shlex.split(contents) + + if not credential_parameters: + raise ValueError(f'Cannot load credential with invalid syntax "{value}"') + + return borgmatic.hooks.dispatch.call_hook( + 'load_credential', {}, hook_name, credential_parameters + ) diff --git a/borgmatic/hooks/credential/systemd.py b/borgmatic/hooks/credential/systemd.py index 0e7cdb47..6e3630c7 100644 --- a/borgmatic/hooks/credential/systemd.py +++ b/borgmatic/hooks/credential/systemd.py @@ -5,29 +5,29 @@ import re logger = logging.getLogger(__name__) -CREDENTIAL_NAME_PATTERN = re.compile(r'^\w+$') +SECRET_NAME_PATTERN = re.compile(r'^\w+$') +SECRETS_DIRECTORY = '/run/secrets' -def load_credential(hook_config, config, credential_name): +def load_credential(hook_config, config, credential_parameters): ''' - Given the hook configuration dict, the configuration dict, and a credential name to load, read - the credential from the corresponding systemd credential file and return it. + Given the hook configuration dict, the configuration dict, and a credential parameters tuple + containing a secret name to load, read the secret from the corresponding container secrets file + and return it. - Raise ValueError if the systemd CREDENTIALS_DIRECTORY environment variable is not set, the - credential name is invalid, or the credential file cannot be read. + Raise ValueError if the credential parameters is not one element, the secret name is invalid, or + the secret file cannot be read. ''' - credentials_directory = os.environ.get('CREDENTIALS_DIRECTORY') + try: + (secert_name,) = credential_parameters + except ValueError: + raise ValueError(f'Cannot load invalid secret name: "{' '.join(credential_parameters)}"') - if not credentials_directory: - raise ValueError( - f'Cannot load credential "{credential_name}" because the systemd CREDENTIALS_DIRECTORY environment variable is not set' - ) - - if not CREDENTIAL_NAME_PATTERN.match(credential_name): - raise ValueError(f'Cannot load invalid credential name "{credential_name}"') + if not SECRET_NAME_PATTERN.match(SECRET_NAME): + raise ValueError(f'Cannot load invalid secret name: "{credential_name}"') try: - with open(os.path.join(credentials_directory, credential_name)) as credential_file: + with open(os.path.join(SECRETS_DIRECTORY, credential_name)) as credential_file: return credential_file.read().rstrip(os.linesep) except (FileNotFoundError, OSError) as error: logger.warning(error) diff --git a/pyproject.toml b/pyproject.toml index e47d0762..61445bf0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "borgmatic" -version = "1.9.10" +version = "1.9.11" authors = [ { name="Dan Helfman", email="witten@torsion.org" }, ] From f9ea45493d0a9ec0aa72af39d06a8e7845f162a2 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 11 Feb 2025 23:00:26 -0800 Subject: [PATCH 017/226] Add missing dev0 tag to version. --- NEWS | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 3f3e5b78..b2dace34 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -1.9.11 +1.9.11.dev0 * Add credential loading from file, KeePassXC, and Docker/Podman secrets. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/ 1.9.10 diff --git a/pyproject.toml b/pyproject.toml index 61445bf0..d886f79c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "borgmatic" -version = "1.9.11" +version = "1.9.11.dev0" authors = [ { name="Dan Helfman", email="witten@torsion.org" }, ] From aa4a9de3b211248497a21fd8d772cc4ff2dae6a9 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 12 Feb 2025 09:12:59 -0800 Subject: [PATCH 018/226] Fix the "create" action to omit the repository label prefix from Borg's output when databases are enabled (#996). --- NEWS | 4 ++++ borgmatic/execute.py | 15 ++++++++------- pyproject.toml | 2 +- tests/unit/test_execute.py | 10 ++++++++++ 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/NEWS b/NEWS index d9eeea57..953daf85 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +1.9.11.dev0 + * #996: Fix the "create" action to omit the repository label prefix from Borg's output when + databases are enabled. + 1.9.10 * #966: Add a "{credential ...}" syntax for loading systemd credentials into borgmatic configuration files. See the documentation for more information: diff --git a/borgmatic/execute.py b/borgmatic/execute.py index 8d850496..ca07458a 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -430,13 +430,14 @@ def execute_command_with_processes( process.kill() raise - captured_outputs = log_outputs( - tuple(processes) + (command_process,), - (input_file, output_file), - output_log_level, - borg_local_path, - borg_exit_codes, - ) + with borgmatic.logger.Log_prefix(None): # Log command output without any prefix. + captured_outputs = log_outputs( + tuple(processes) + (command_process,), + (input_file, output_file), + output_log_level, + borg_local_path, + borg_exit_codes, + ) if output_log_level is None: return captured_outputs.get(command_process) diff --git a/pyproject.toml b/pyproject.toml index e47d0762..d886f79c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "borgmatic" -version = "1.9.10" +version = "1.9.11.dev0" authors = [ { name="Dan Helfman", email="witten@torsion.org" }, ] diff --git a/tests/unit/test_execute.py b/tests/unit/test_execute.py index 8de49912..c9ca6e77 100644 --- a/tests/unit/test_execute.py +++ b/tests/unit/test_execute.py @@ -554,6 +554,7 @@ def test_execute_command_with_processes_calls_full_command(): cwd=None, close_fds=True, ).and_return(flexmock(stdout=None)).once() + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') output = module.execute_command_with_processes(full_command, processes) @@ -577,6 +578,7 @@ def test_execute_command_with_processes_returns_output_with_output_log_level_non cwd=None, close_fds=True, ).and_return(process).once() + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs').and_return({process: 'out'}) output = module.execute_command_with_processes(full_command, processes, output_log_level=None) @@ -600,6 +602,7 @@ def test_execute_command_with_processes_calls_full_command_with_output_file(): cwd=None, close_fds=True, ).and_return(flexmock(stderr=None)).once() + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') output = module.execute_command_with_processes(full_command, processes, output_file=output_file) @@ -623,6 +626,7 @@ def test_execute_command_with_processes_calls_full_command_without_capturing_out close_fds=True, ).and_return(flexmock(wait=lambda: 0)).once() flexmock(module).should_receive('interpret_exit_code').and_return(module.Exit_status.SUCCESS) + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') output = module.execute_command_with_processes( @@ -648,6 +652,7 @@ def test_execute_command_with_processes_calls_full_command_with_input_file(): cwd=None, close_fds=True, ).and_return(flexmock(stdout=None)).once() + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') output = module.execute_command_with_processes(full_command, processes, input_file=input_file) @@ -670,6 +675,7 @@ def test_execute_command_with_processes_calls_full_command_with_shell(): cwd=None, close_fds=True, ).and_return(flexmock(stdout=None)).once() + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') output = module.execute_command_with_processes(full_command, processes, shell=True) @@ -692,6 +698,7 @@ def test_execute_command_with_processes_calls_full_command_with_extra_environmen cwd=None, close_fds=True, ).and_return(flexmock(stdout=None)).once() + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') output = module.execute_command_with_processes( @@ -716,6 +723,7 @@ def test_execute_command_with_processes_calls_full_command_with_working_director cwd='/working', close_fds=True, ).and_return(flexmock(stdout=None)).once() + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') output = module.execute_command_with_processes( @@ -740,6 +748,7 @@ def test_execute_command_with_processes_with_BORG_PASSPHRASE_FD_leaves_file_desc cwd=None, close_fds=False, ).and_return(flexmock(stdout=None)).once() + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') output = module.execute_command_with_processes( @@ -769,6 +778,7 @@ def test_execute_command_with_processes_kills_processes_on_error(): cwd=None, close_fds=True, ).and_raise(subprocess.CalledProcessError(1, full_command, 'error')).once() + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs').never() with pytest.raises(subprocess.CalledProcessError): From 8745b9939dfe8e3b4a5ecf6765682baf2cb660b9 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 12 Feb 2025 21:44:17 -0800 Subject: [PATCH 019/226] Add documentation for new credential hooks. --- README.md | 3 + docs/how-to/provide-your-passwords.md | 164 +++++++++++++++++++++++--- docs/static/docker.png | Bin 0 -> 11554 bytes docs/static/keepassxc.png | Bin 0 -> 24198 bytes docs/static/podman.png | Bin 0 -> 16436 bytes docs/static/systemd.png | Bin 9944 -> 10149 bytes 6 files changed, 149 insertions(+), 18 deletions(-) create mode 100644 docs/static/docker.png create mode 100644 docs/static/keepassxc.png create mode 100644 docs/static/podman.png diff --git a/README.md b/README.md index 4d0fdad9..925b2b33 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,9 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/). ### Credentials Sentry +Docker +Podman +Podman ## Getting started diff --git a/docs/how-to/provide-your-passwords.md b/docs/how-to/provide-your-passwords.md index 01bd6a9c..b4b5a8df 100644 --- a/docs/how-to/provide-your-passwords.md +++ b/docs/how-to/provide-your-passwords.md @@ -19,6 +19,7 @@ encryption_passphrase: yourpassphrase But if you'd rather store them outside of borgmatic, whether for convenience or security reasons, read on. + ### Delegating to another application borgmatic supports calling another application such as a password manager to @@ -31,15 +32,6 @@ to provide the passphrase: encryption_passcommand: pass path/to/borg-passphrase ``` -Another example for [KeePassXC](https://keepassxc.org/): - -```yaml -encryption_passcommand: keepassxc-cli show --show-protected --attributes Password credentials.kdbx borg_passphrase -``` - -... where `borg_passphrase` is the title of the KeePassXC entry containing your -Borg encryption passphrase in its `Password` field. - New in version 1.9.9 Instead of letting Borg run the passcommand—potentially multiple times since borgmatic runs Borg multiple times—borgmatic now runs the passcommand itself and passes the @@ -48,9 +40,9 @@ should only ever get prompted for your password manager's passphrase at most once per borgmatic run. -### Using systemd service credentials +### systemd service credentials -borgmatic supports using encrypted [systemd +borgmatic supports reading encrypted [systemd credentials](https://systemd.io/CREDENTIALS/). To use this feature, start by saving your password as an encrypted credential to `/etc/credstore.encrypted/borgmatic.pw`, e.g., @@ -146,13 +138,154 @@ The one exception is `borgmatic config validate`, which doesn't actually load any credentials and should continue working anywhere. +### Container secrets + +New in version 1.9.11 When +running inside a container, borgmatic can read [Docker +secrets](https://docs.docker.com/compose/how-tos/use-secrets/) and [Podman +secrets](https://www.redhat.com/en/blog/new-podman-secrets-command). Creating +those secrets and passing them into your borgmatic container is outside the +scope of this documentation, but here's a simple example of that with [Docker +Compose](https://docs.docker.com/compose/): + +```yaml +services: + borgmatic: + # Use the actual image name of your borgmatic container here. + image: borgmatic:latest + secrets: + - borgmatic_passphrase +secrets: + borgmatic_passphrase: + file: /etc/borgmatic/passphrase.txt +``` + +This assumes there's a file on the host at `/etc/borgmatic/passphrase.txt` +containing your passphrase. Docker or Podman mounts the contents of that file +into a secret named `borgmatic_passphrase` in the borgmatic container at +`/run/secrets/`. + +Once your container secret is in place, you can consume it within your borgmatic +configuration file: + +```yaml +encryption_passphrase: "{credential container borgmatic_passphrase}" +``` + +This reads the secret securely from a file mounted at +`/run/secrets/borgmatic_passphrase` within the borgmatic container. + +The `{credential ...}` syntax works for several different options in a borgmatic +configuration file besides just `encryption_passphrase`. For instance, the +username, password, and API token options within database and monitoring hooks +support `{credential ...}`: + +```yaml +postgresql_databases: + - name: invoices + username: postgres + password: "{credential container borgmatic_db1}" +``` + +For specifics about which options are supported, see the +[configuration +reference](https://torsion.org/borgmatic/docs/reference/configuration/). + + +### KeePassXC passwords + +New in version 1.9.11 borgmatic +supports reading passwords from the [KeePassXC](https://keepassxc.org/) password +manager. To use this feature, start by creating an entry in your KeePassXC +database, putting your password into the "Password" field of that entry and +making sure it's saved. + +Then, you can consume that password in your borgmatic configuration file. For +instance, if the entry's title is "borgmatic" and your KeePassXC database is +located at `/etc/keys.kdbx`, do this: + +```yaml +encryption_passphrase: "{credential keepassxc /etc/keys.kdbx borgmatic}" +``` + +But if the entry's title is multiple words like `borg pw`, you'll +need to quote it: + +```yaml +encryption_passphrase: "{credential keepassxc /etc/keys.kdbx 'borg pw'}" +``` + +With this in place, borgmatic runs the `keepassxc-cli` command to retrieve the +passphrase on demand. But note that `keepassxc-cli` will prompt for its own +passphrase in order to unlock its database, so be prepared to enter it when +running borgmatic. + +The `{credential ...}` syntax works for several different options in a borgmatic +configuration file besides just `encryption_passphrase`. For instance, the +username, password, and API token options within database and monitoring hooks +support `{credential ...}`: + +```yaml +postgresql_databases: + - name: invoices + username: postgres + password: "{credential keepassxc /etc/keys.kdbx database}" +``` + +For specifics about which options are supported, see the +[configuration +reference](https://torsion.org/borgmatic/docs/reference/configuration/). + + +### File-based credentials + +New in version 1.9.11 borgmatic +supports reading credentials from arbitrary file paths. To use this feature, +start by writing your credential into a file that borgmatic has permission to +read. Take care not to include anything in the file other than your credential. +(borgmatic is smart enough to strip off a trailing newline though.) + +You can consume that credential file in your borgmatic configuration. For +instance, if your credential file is at `/credentials/borgmatic.txt`, do this: + +```yaml +encryption_passphrase: "{credential file /credentials/borgmatic.txt}" +``` + +With this in place, borgmatic reads the credential from the file path. + +The `{credential ...}` syntax works for several different options in a borgmatic +configuration file besides just `encryption_passphrase`. For instance, the +username, password, and API token options within database and monitoring hooks +support `{credential ...}`: + +```yaml +postgresql_databases: + - name: invoices + username: postgres + password: "{credential file /credentials/database.txt}" +``` + +For specifics about which options are supported, see the +[configuration +reference](https://torsion.org/borgmatic/docs/reference/configuration/). + + ### Environment variable interpolation New in version 1.6.4 borgmatic supports interpolating arbitrary environment variables directly into option values in your configuration file. That means you can instruct borgmatic to pull your repository passphrase, your database passwords, or any other option -values from environment variables. For instance: +values from environment variables. + +Be aware though that environment variables may be less secure than some of the +other approaches above for getting credentials into borgmatic. That's because +environment variables may be visible from within child processes and/or OS-level +process metadata. + +Here's an example of using an environment variable from borgmatic's +configuration file: ```yaml encryption_passphrase: ${YOUR_PASSPHRASE} @@ -214,6 +347,7 @@ can escape it with a backslash. For instance, if your password is literally encryption_passphrase: \${A}@! ``` + ## Related features Another way to override particular options within a borgmatic configuration @@ -226,9 +360,3 @@ Additionally, borgmatic action hooks support their own [variable interpolation](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/#variable-interpolation), although in that case it's for particular borgmatic runtime values rather than (only) environment variables. - -Lastly, if you do want to specify your passhprase directly within borgmatic -configuration, but you'd like to keep it in a separate file from your main -configuration, you can [use a configuration include or a merge -include](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-includes) -to pull in an external password. diff --git a/docs/static/docker.png b/docs/static/docker.png new file mode 100644 index 0000000000000000000000000000000000000000..a3a192ab82d50e8b3d01a127bc61e499787d507d GIT binary patch literal 11554 zcmeHscUV(Rw{Jr4O^S4g^cDy$^xi}|ib@S6ASDob=v5HuO+cE63W6viRX`#ZiU=

?dN;iFJNG=#eZJ@1^WT?bXRkf8<~M78vu5onN#>>ov{W2aAP|Vw$WYf3 z1OjIOGM0i2_@9`5kO$m;LtQm;H8K(dQ2-BsCYS^S2BZw&5B!IW1)hn(#2{keo&a1- zfPAWRdU1eB&+aLJ%=w3$1IQwO=nw(2444Q+3*3u<3kJx{z`Y8%+!KGV>lGlMekjhq z5~no-Pk-JGj7-e=U{W&DQnGNMyfjQ+SsJD+1LK29E6YeL%gY1W;Cp{tbc)ntCioCQ ziF4P{F*nlD;lud(xVn3xfTkBkWU3jy(d2sPICL?&9{xq^Ex*n(l8wdp$z>7ko95#6 zOHO`^Y;x{KmUdjEqp@+k$(PC5-0ZpSNVV38-`Zm=12UOT@|dr`#bPGXgLip?S3za= zE_c}tiSBx{j|E)pW4rQ>AWovoph@9dP>`x*)Qj^EpM29b+DbC8p}xCtt5Po;tey$) z;YiSH(^7rmE)QQlXK+pLO42}*FwIN8^$VjcdDHMbZdR=8a5ripsr;>TOM0%lD~ryB z5}~xmcTIn3PoZ)*S>0+rBPYW)*bPt7) z(v>5V?5J9*oW5SwWqVt~Nom@?>~z5yCP~`f)^vx4s{SuBkewD1!N9JPSRktk@Ql~xE^LGAWKUU_p zz}*%$1)Kx|!>v5gu*tc}vg4^^wV~xza&5Fg67vgl3{glMQ!sbZA@05mE9!G}eAdHXA4)gWiM%7A>D42AHWsRVecL9POBq~qg<;**n- zlaiLy!@38-AnH_ns(wgUWlLTCKOlf7HOP&C0E{ve8XO!f6)Y>|jtOx0@#Z_lL^%5d2B<+GKtA7}_MtH*CV#rW&YFXCfug)nXh78;s8!})syrQVf0a0;z|9?vIV%N_{V$pU?yi56^)I%a z=A1R>pN0U%|HAzj?LXUorVMDAm?-P|xCEY>XQZnJIjvtA>Eq&#R6Y|CGDumNtcVKR~^S!oxztO7zo9x4A12s1x-z$+16|Fr593JE|dAYpL0ysM(5 z0!khRSfhZDbar(H>~dC=fg=?V@-p&DXHZBNWqlt%Gy>>OcQnEc1;u!~on@R7uB>Hl zqy~XWN&lTeI=$~t~1M1YTCOZh+E^&h(aEe8HA<^Oos z|2Mj*{$5U@yn$IzFtAuEy`s4VEVRg+jSY0qmRz7;I;~NFM2RuH<_`i%T{ykK*oPWH zKq7g7k%=DpA`vYKm7=tF8XXA4g)`FCvc`@t7rW(iy(@^G<`E#dg=AUOY_E%_ydw?a zEa0mnC5or_k2XqR`V=FrAf!@6D-@^XMaOUtmd{sbB2AITN12&+LsI}8JNaIro7pHv zV0j|Fs-06(1&?^?<`&ek8!|uW@u<4B>Sfg{Do2nO1IhpB_kEe%^dgno@yT{ZL|T1I65TLV%0EJZmab=_jxjj72dU<6QWfT4WCpfA5g`U@uuj5 zqd*Ez_V&$ijF}d^#+K@y{xhDBJH;N{P`xdxJQ5$ZJ8bLLTH*z>k{mAK4OF4OS?ZvAjKCW1D1EP++Q`8|!1}C;E z^4P`OZZXv2#~o@?LWa)Z*cZR85Nt(L+9YevH~axgC&Pq&l3eGXLqBfsGn{s`A^J>whm@7L2G{kfW*{l$ zuB}Nl#ji5hBRuJii{Opa3mO#aKG1a1o!xI&#Sw2#&gX+Cm~q@{5p>KRtyPXaX~aPS zh{N0F_!b;DPBPOXT@lB(Z$i*?qG$4c-1sK1tWh?)HRkcL-SYyHuHHzNsH^Wr4f1Ak z&G9ms7U%|bmu_t9rLY3E2$ks`>NJ6nLUL7#3G3L7ucDPY=%`pL=PHTK!m>sc1O0|8 zM&;2_#5?ZL7jbOojY_t8Fi^~8;zfOfzccT>Hh(;C>eQsqheamzT%vpHfJUiSs!< z!c3DJ#hEAE6N#YABC5*4dkZoAq&ag=gh8U4277r{Pkkmezz z*PJSs2UBa8C>q*Xo5gRgf$~dd*=<2)EF!96W87HSa8Nak``pw4*6F%dXL;OnpnJZJ z@nGj+xE>Vt{Xo&}@6F}m6&%;Q{Awtk-0@cu0*z*^MXL%ol9o_3eAGL?U(*!k8#cCm zpTv&IBg~neD!m zqB{7ZlEYV(_SEY<^W}RRyL9iQ4O*HdfMh*cAS|8b71~5aEFP_<#nbpvZS*tZl|Fdk zu~C*JL?e3IcA-a)TpYBUFz!8#oLur{IE4St`eZ$s!u-f z)ibW}$t*W^Ek~0n&^u`wC}9#4Rfs!w;%qq-1j^?I&GAFBG*c(I$Dxt*=LezpzD~cZ z!@0s~A94DU-88!P$h11`apXqUx5IAGn+xTS1aB-5`?`^eKMH?;0r$9B^5W$|buuuX zVD0LLVt_Q=9H*-qYF>|7@?wEhixFMdd5f)h@TEw_3ZL}>1=RMDjtcY~Dk ztcQM5`MmzTtKGhAvzQzetXV2VvTBce1}QJ8u2Z&+-@JP$hk4%A0Vl0kwCcV$e1Cbv z0WY@L=f`L&xKq?EV*BkVsM1PG}s|y(rQ>6XY=M z<*9dhIR9h+V#WJKU&-G#vwRBEQ$YGmODmNVZV(IU>QCm~FLsQ5QEb@T0Ou^Qsj5$> zkLfyGQ1$r|BDTKt{$$jv9VB6S?b|BO9c(w`I_y>)wWY!i{WV~Zd;E&l8yrqG!O6aT zr7NqRd%+bVYW!-SZs~4`)PtFS3%o9-sa5 zW77?ldo(rzef`da(TCCgpS}$0>PL_iv3ndCXk3_V7zc4T`^Q2^y7(NU$cJk;H270v z94ePhCD83t-2TyV0J)}P>JDH!+J^SGO`8bHtmShatv<+MG^zFWJ8)6f=gl~u1^VD) zbxo1`vR>bt$n;qcE3bjWn?6>W`S+jDWGQtQ(Wcal(7ofOo2E%IDcHK*U72*bw^m4M z#W)LmbQ?vKRu#y?=ApvXlvnb#c0yM_)eJMxzam!k9M_SY=#M>d`*qEKVYxPs|Ib{B7CsHmHu0n{bhn>UrE$!%O`7f@84oPg{TD^9~;qM23S9 zy_8FAk%2p1s$En{#67`;N&exT(2kqt^fS-Hz@09w@wA75P60gnDZrdVj5Os=x1g=tjjvV*TkXBLWF*1o`IGc zTB?S2#tLkf9@dU7rCUUm&QL^^hr-E==hZl*oRzG9$Hg`FLt<`mSaA{S99MqR+V?M= zOt9LiOkR@GhCBnT)^oLf>hI%$p@)~jTvi#xodP9E3&gE@`VkqUI<^&+f|BDajzNiQ z(^bAs*V*Ba9zt!n=-#fXI22YjVtamoaH7U&Q`lxgWcu^bD6b*RHvecdsBpH)!4!(A zYoK3wav_<3<%GjvOEDaEYvm-PZOpi4RW-_b_HFm%Mwbu$u3ks^AEIdN3f{PwrKap# zF|Iy$tX$SJQNLqT5+kvGa?*KnP;jzhT;yuXkK;6Ghed`n`azUzgvEc{6XPBod6S z?XCposRN+MbN-8ElavgEHInqgqMgIXi8hOIZG*0le{v_2=O1SG7mRhAvL5Qt82b$!*`;^h6)de}cB}LZKkTm*pKjT@+WnfwG zg}iAw=bcNZZRE(h6Wui)tjT$l$6lt-9%e&(7O`J=02}L zWQteUN_@HwkDIzHTa){Bk;84~_zptjeaq*~eXMU)?zTR?BbmWmo$(m8uvXi;T<>8k zYV`A>(C&shjI9EquC{RSeq>oq@2RQy_%97@^y-fIiHFzc`^(WO3(_{s_6d=Z?#*EB z_Lhds_{885o(0{NMrGH<_Yy?$$Z?o4kMwO z6S}>L#+BsH#p~*<#6pz6MX-_eC)&>tA;H>a7^Apv!aV}BgUa&@&r=p>b2xjoVDi0f zSh#*PjmKb6iY?%h`t)F7Ctk-2;ezv;W@upkRU-v1ieuj@w{)^!fdGS2cwJa|R} z_7+!mx*nG;C}nDYKj%(2PnT`?d)uQpFP>}ELZ4S4u&Sx&rgdL#yydPk&%uI#y*jz1gJiqM0BUsN^U)LpOMH-*;-FXfEih?i1r2bNtAUq6&%e7;5V8 zDSFheAI4wmlx%Fc+s51X+Wnp&3yq65T6lL*W-c{$++K?4t6qHQx;My1@y68DC5(x2 z@ur-rPRX2de^sEAj)lx)8_(HDj25U+Utf&}?s;gy^r-N>hi&zYe@oBO(uIv}#=K`B zN+zxi%9i!Qr-!SG6h=QG`k%#R%Nl>2iz*q+Wt^OL{`7>?qxB;95BbLa*r&u+4etf; z4=C{`KX5^r7Q#Wz6zeC)p0OK27>ml~Ncpd21^wY{;JNUW%!CJyg%-&=Xh)F3o%MF} zc{1sT2sfrRXQJTx)(rKRj#q+AFdvQWNBm<0%6$h{mbC1OJZ;~VZ{Pz~StBc4e}KSV zpK^U3cUMLwjITs*(&h?qOplZ`uIlTfEKsJ-DzfAY!!IhX{@e@+QKRATWa2xXHqeb` z?teC(vVY$~cGP1y%N(D#7>U1;&Wg?(z~2mK$=Z9dbTL{rn{P5^n}H9QvwPrbc0nxizNS+s&ZK&n> zuqOo)+IeI;Yr||H&=wx(q+hu_3R4y9C-X#BE{m`?W>IIbTe95zB*BxO9WQnf)1dpkMAHbx}}LkfbG=!mj@ zmKAbDk$+Jc-Z5yenWGUQHOQP`C(vu7p@#hT=D|(EAsW7jM7?P1zMO~3ehY8AD|hco z40M=Dt58R)Y!tmdkWD%;WS!k=nmqbG_edbAvr!6TR^koQMmGtMrC=E`zI~dzZdon< zEx@aOT9s3)a+wZT(It#K@OXq>s$9Of?G^81r&P%W-<~=#%Ky4&j;9pmS;v!%KB2ip z-$1)5W|Q1_q=Vkw4h-|CL|chNbHzA3#r^V2gz0tl?Z>;;p(KkBHqr}$T z_(9@e`kUh=CEHGF!&l~%M{47Z(ma91>^T~B>P7XbJxgs8tio{6Y6|MA;P4I=fo%4& zIe3aVvKVu(wFK$CWsG}R*4VWFW#?M&bN8_p+SOk3XM!`0W>e?x21idC&gayTOt5>j zUhBl4#3Nq~CV&P;Aoq^G&fXNuesC^4WlNutM;P=X#>5w2*+IV2b@N6EFk`g;cIB<* zhw3_6r)$JztaK?UFP|4sGkAoFtaZsnaKky9g>EP9D0;QgzLIzWU6Y+Q%=R<6eiY8T zj!tJb;vq9NV^WtNzA`gxY9?fVc&q(F`N~C$gzcCGCMx0RUy&ngDjn0jD)_C=9n9=V zT2oreeyP-oOiX^zJ?_}8|&rOA*cVV`%2ggisHN7jY#0z8?f2LNc4sDH8 zY}}_rNO_Q{1uwPB_?vhM6+Q<;hug^-8V|F>uRi@bwZ0qk5qJ+#zifWJjw7YG*7B(7 zJpavc2l_)kRNYM+apdPi*LOQ__?*;=h}$PIp@;5R$>@(SW%a8q6B;lF-`$U}?Q>1= zlDF4{qGDAzuhRzy)k*flI-+_H9GKYy?Gl35bqG~CJ-w0haOwrAhkoKWh>Hfld*kki zu8e)dS=WQs2jE3tOBfs|2r9m1$)lxfT5F408^jNqexBUB@9nsu%~|xKe+D;?TM1Y- zaS|DXa^T3>beGsJ6f8(#q~j^J;wr= zcrGmgZ|G>FMF;-YUuF*VJJw-9---G#h=cvvPTk15&hef;K?Nr*R;3~sGxt45Dcd)Z=_D!?s_BtBB z)Z5Sxa6$u5-+Q7WotDoWWF^Ak3w!nCA7sR%)!GxTe)i+iISS7Ve0LCopZy^NDeo8f zb!|Bf1&6*?(|8-Aeq;Id{mE&NeWbCIQAJHhjSgD)7mKF>S2-QazE4ioC0nWG8LjfM zt3SzGn!;>fL&~45+L`U(=1%B-eicMUbn!%1Z84aIAat%r^E_N0-Sw^?;eZBr=Uw+eD0tGZVuuFIPAM6pV z7RYL|Z7!P{ueb@tyg=-$tWlqgaN+hle2B>s#-GNiEEJEs4$oTl(MDaW+oOueJ-MBt z+yrHT!orScPHGBA;)`Gr=MT$iX)+DtC*L4JDk2PdE58%$3#NCTipgd zAsBA_E8S{-Mxp2+v%j7g56rKASbrl~cCTfw}8iSVHTbCOZ@ z&s)S@=bf0>6?jTR%VfBl1z{ehaXBvHPEZYVuVSYkb83!htugraPA#)MgkaJ8v0Y4J zY1}!1un~IP5TtzN;5D=9ilXDZrLe^`UW1cc#08`)S4Yd&BL#R)J0Y zDRgXbS#%kkxI0&wvLl!XE9&#H${fFqxiiY@hBz`R?tNJowZi9grL(5#dWj;9Bs2(^A`e{frRm|dlT;awgKvV&3s;e;p*HaIf7p$V3vPZk9anGl|-8@wX z!nHP%yXfw!mdRE6XJ9vGw35W>dCP+^tvZ?hyxvF?^WzE(IP%d&E8sNjI|6LUZ>WGH z?NEw;uY994-CxO9kH@={^z?j%Sr?cT<{>1c8eKvI@OYG|`p^K=H@VZ;gYB;q%H zhdp2GR}p^4SAnDEc(ujBsqOnoWQ&!d{{QIs@1LiKm24-}@~tQVbx4@X>FKADo~iCr IZKv4(1CT9lr~m)} literal 0 HcmV?d00001 diff --git a/docs/static/keepassxc.png b/docs/static/keepassxc.png new file mode 100644 index 0000000000000000000000000000000000000000..059e7af9f829728232ef4b78f2f6fab34532c282 GIT binary patch literal 24198 zcmcHgWmFv9^92eAg1fsjxCD0(HfV5ncXtbr5C(Ub;4rwmy9al7cej`4_rLF_`}uz8 zS!bQqJ$<_N>0Py}szE;#q>vE^5Wal*f((=vSN`$^V)%0$3xjj?NC|R<>qezREqeL2PtQ@=H5TI|8B4p*({VN0)Zu6q9_1C%(L2%J-DY9udkq|3#3t#@#t6>9QY=;0u=SRx)mD3)iKAVi`~ zRG`+1ljj0cN=rf=Gzp?XY9YI#T)H5Ce!(>?X2azufGeyCN3!dD62Sh0M>&oXOtSeAn z0`>?J8Hx%}IQG!~Ifd^kq3QbF!OqUi-u26OXEP&LGgC4TD_2W0DWLohjX)IKFJH*M z0L4XBJ(o{2J$$j|R{A^HYnjr;5a547iB(RgBR~deia-<6E{aH;hDB1m{-DbZ(#tP4 zgkT8$NhYo29jjmofq@~7dSvHezAnsVXW;=9gM`}PC8dlp^|)AVcA4~B_JUGVQ*%B1 zm-GD(3NbU<9o_$p_*~;=|2Osj<=_xd^50$gT>sw#T>`v{!fr5FMn;B>j*c!vI9owM z0p;oG2@3~@Y&2}83PSXL|PiIqod9 zZ5d-@|M2?1@>opt#U&-lsH<~%c{T4I;2JjK8rLoWyMn{9@jw7?P zNy*8{Ijm!nAe6Asp+pVPKbQM2Cl4wJ47QfXm($lLSz21k5cK&x zwc&D7H~+wlnVPpPYoMOLNv)t6&nhoOr0T<&uu$| z(EhJZqdgQ9)b`F!FaY>G@_**%3EE`fR*9&=( z)1}bRcB7X}`%KwP*iu;t#)`pWIpsftf*=tP5b1^{X|8NBq)@ZSn5id*dt5b>twF!7 zYO!-^5$u+Azb{ni9Zu!J67#$F*m9jPNc<1&xVGCEv((d5+xbd^$r`iK&@etxeSIOA z;GAE0RKT85>+WNd{)NG0o8`tFv)29D5|!i@h`Y<`M+%wT@YcDjqOhGXnGaPgsR(u^ zhTp&HJhvzMsvTY+a`W)O0nSY z@Yjfvf4zr?nOAs|31oeJ1^zq`lONwYDUb-uJB!ihlKtJT;;`f5&xQL6yFL*^{@rT1 zuEbvJ_&+=q;^u4wNq8K83co)F8~8uri=wflj-fdBZ7J9VoN>CIucpqASCDg*F$enB zqeEBI?jCxD;?|swwAO7eESnMzTY>ueGK>Oth2HgFP~j~wVv#WDqkd(tuIh@3iFuw@ z4<3CQ866${`S||)=dc9vq5Yl` znG}ZF0a-CQqMMaVzq@}BmyywzB$)oVlPnCBSCCLj$o+<|m2@;;PWZ6u@=eg^zR$S& zuvGg0P%LY^^E_Q-HJ{8HpUC9@p~iwo)MweCX*pGc&y8|8?ndmry>k{0kV%YsP`iC# zZ~;XzYGJpoDdEqII5Nb+R(_mIXheOe9A#UYwJY|4+P2MFuZy7!~>sUmt=nlV^>oNApf5l3aNHD=2x&LE68sMZf z5NPytz2|$?kGpu*kh_={x@4R)SuWUIzH!E3yNLEJehw5vFB2C=5ucPPqiv3mx|F)R zw`W{CN?Q{|!4Rd44gmo6nR0FI>_lcq3%fyzT~#N110GoMvDM5hxojHv>$F7V!A2?R z+GuH7=4sWM3*61B=z6TVrN~kHsU7{_HmQ)7lQZoOf@ajZW(dnQ<{*d#fR@Qck^CIt z1S`*gOxa%=e`zYk$xZT2u9}fzBsxYA@dgXt<0bl0! zN#&%a4JR_bJ-$6`!~(ub=E2u4N$>1OdOG$w^3|V*0o3)*$IWYTdRp1* znvCh;4^eRmHE1-VT*!5ti{mF>DBm(6q>d-%o|9iUJKNbj{*O>W zWE7)p*#YmoTBh(rnKiDS6yB0dp%5JyI4g+N@{=vLrGnmx^h7> zRP{po*Qk8}+dDhV+i#qjo#{J28AA=bC?C>DoGwYup-&W_Y-*Cx8Q5)mOcHUyy(MZY zPiiJdM@Kc0LJ3N<$q`~SiHTBJ*ucQ(KgzlKNCd>k$3LC7UOc{BM5c_JBf)&d`NsZ$ z(i)!gASB{wn2AWgM++8H*Ox31npqkB!#rNm3rG$BH4Y7hO%umhO$#le27d)#WRiED z_)OGlOVdd95F`eZOE#4Yl+E(prfYdP4+8M!-yt9(rgA$lmugfp(-Z!W6No=5rPv+)&QKWJ=9$f2pwu77jlVgMUN2s914>H6!=vVB=W4#~)(U$tjG=Y5t7#{c zFl5CP#q{9qLntw&Kd5R-N@hQ>q2o=B9`6*lcGQ~9N3HNnd^>!pX?7G0#!2BVr+d)=6uYRzk;{payt1IU3|NuRF)a<=a^5Cm;?SIzdg zE{fE2gwn>S4vX*Ym@@##Fzz1llGD1T@9_68*wqqEwy6prK0o8kKr!O}VL2(PM7)P* z4Q~Xd1rx_x?sZh;H8rSxLsH)Bse}m*+`SoYt2u<{I(-X|$Z(C${{{z%z7>hb<%+l9 zY8y$$^K03Ti!j^&nJzXq_E~1;b6Z4}qD}7($J1!sOMLodv;DDD+OVK0%TLGWJY)CP zQR; z77H3nqMLce5@$MV@VR3I+3sb_t|SFpD=8n*?l%!nbF@hb?4pY~tQRjd2V7LsUkQRk zQgKajJS7^%nAoI1G6Yjo)0m_r$xk1Bb$2(Xk@27HbR~*i-ECA=u@G^Y0fK_Wz&JFC z*og)A?f0DT@L;u-ofF%8F|O06aA=!OG6|tSa|^H%iC1i1&1h=BIkc2Y2(bn>NxmN~ zSbd?Hkcdn1$H_CzwX$S5hMpfZzNe?>FY%=-CWFJH`or>5o@qClvJ=fiD9r-4O~%qN zWl(!^KQfK|`U>6mdaDS~PeTc$Pv?63K2s>~^|h%aL4WZeS<^-BtQWZ44M>rNoZ=~@tgZUR*eMAOi!l>2ME=a zQ2;UY)e*nKtjiQN=*0jYaMJavZw~Gg$Ml2|9u)sodDvhC<)&$1~ts2Jo zxcEarJ~I~j%h9cL$MPk9^{~Y%MnRa&PY1GK*fohAb7XWVbRY&OTG8-my5KWFK4wt- z&qcp8{P(P%F$bkoG)cFi<18eX)l{fH_kGXG_Rc+L7)9+}m#}ZE6$-cjZCG=Ff`OU{ zDmOkiHjzPtVha?3h=9P8>5eDL2)<>czR#VKqP{SnXi3etDDv{%sET=B74&7Hu7wvx zQ&LO3J6Wi;m?ZJr``-hr{Y0QTC;E(cKL3dOLyT3`%qE}hdc09poc#?fJ3$2G)~0sd?+}cI|PwoOHd!LvYMWT=Ki2a zJMX)OBMRM^@JK&kubG^&XSb9P&6ho~ISLpwO8R(&jRVA`q;zEmR1|vM>_fr8=m+XB zV?Irhou?B^Eaic)-f1@~NMwtw&+iM&&n?zn6tA|TI5P5@AVa*lS*^}}kgUKbtLkuG z3|{BU$yz!;=D0D?#L@GHgOU}$bi0YcJIp5F*e%c8EuS1svq6QgliY!WE4B0c>P+UtfD-ucmSvrg|>znALJxwm8k-qK304Haw(*Ymk| zx|GH$GByTE%TO^<{%I2<1e^AZ8ux26gtpxEz}fsuQD3Ml(i>E8wZzdqs@FSDN=Z*2 zU9Pv`4R~+sB~VuNUh@`Q<8ynEX0uwN^R0D9x$>IQY-2(RQl%r8#D)-cLo;h&%A~YF zQ5L4{*?Cdp*r^BaYC5B0{T~0rZOr<_gKm0-7q)i^CAxiq$pC`09qVHDAO}^WDV8-#*^; z>^+#k;Ars!JGJMP2;s(MNE$?We96pJp*cn*+QXg8|20IY$K<=uR2R5*Zw``mSDjjZ zX*g#}7|rBc?I7Ket$%<3BA;Ks$@n)t24PMgT++zQQ6pqgisvc6F(gS15ZPJ2M8#fm zc2dpjq@8$XqgRDi%*`z4l4HrNCYeDG-|P`Z^@+z> zj4~xJf(eSs4t)Tbil#4D0W7IrO}i*dN`9W{4|mP`C8_3X1RMp`_KpH;JI2Y_Q~@QM6X_JF2%LDxe9uWw zfAaAb59y2+DF&ZI`I9V(Tw11lG(Us=wK@Pa2mr)S$qh0S7jLj2DPetz$JfuO@pPl} zL-?t9c}Jb~Y6@4&m1AKq@XF=`Dm#EPknlwMIZM5_TwZ6}eZSL_6m{}39iXRmMIV?L zH94`l6t$j4tW72COtK@q2vn~>5T^%xJaNVQ3kH|*#W!Gag*L7eiMNM)t==fqI3EYE z3YiKRIDjZS2?opQc{7$?Id?)4ePUj35x`6OB6YCjL56xV2>i4GR7u z2^mR@lMg-F4m#psQD^BAoYnzY){*dgdvN8`rGx_ih&-{c^M zdnrMGAUbcE=V=XUz5SeK{TNfkZB6Fs)IrYM27CEOv7v2G@XWx>0|k!!R;rqupFf2# z>9{1_pm!3Z|F2)aF$q)=OXq%xG6TMS`xCM2k|CHZ(;$Ck=>>M?`xB5g^v2J9a}2A3 zKfklI8+%Q61moPiAQKT5T0v7|qGYA$tpt9Qu4MtvkXl6u+Jf%YHQ9Qg-jh<=GJC%q}J! zU~71aPNsVLF2z|E2J^t`HjQ%bbBBttSCfx68V7W6S^IXuPL@w_O@3OW$+1QJt}yc> zb+e!I^LDapehlr*H&MQ$hy?PRXp1IOgYq zC+N&a_(|d7!4N81kFg3doXd~JYia2unPWYPoopu3vc7fv!OmGsq;flro)Xz*rH~Zl zjq=-PK9nq$nlcL7<16w0R23u4w~!PE(L2G!%WYrLJI}veO&r=3mqm}^ykVLMpywL` z@=t=>?$)&DKd#|Tq>y%>lW@tTzYHgP+j%<FErvE=+28qfP#wB=baXOAaiSuO6y6XAto(Xk%}^mBgUG7hW~p;k6q?*1 zWOKzAP}W#ekl=zsbBBloJYyaoA4?yNRHawmArvE?Lo*NRi-3so7wN-29V647#hTNnjCIH=nn+U~K9UdKhn| zdf_u}yiw|}>yuf7e*817Pof&j%?6R-L8*#vMZa~{?ERBOI7>cI!h#A+h%m(|e+uz~ z*J06#goEkk1r~*V8Ew(X>|beFLEu6}ASTDtzTj5U7tDRV<3Bim)N z#ZvnsRq&r~1@0}Z;goN;(xikwbcrwJB_r`$U-f$cCj#OOd zx>_+OCp<$?)|W9Aaax54CfkO>Z&piEby9iH8KST)kI4(XSD*INk-)ZvQE3w!>;pIQ zs)2~Tq5he8f)1<~$PTjQ)+H&JmsZ{kCi`y#5Ao;uC2C)~ll7wo`#1*>wBxR>u80J^ z%R=$Qr37kRG#aagMbQ?$O)0IOKzL(u41rj&(pMW2+`^dQoq)ix9#foG_H?`0^lAg+ zaAh>8xCxpX%Q-Q1BF%`#PSm(q3Wn(`HCv@H;}RcDM|u;d24=IlfgG2QxEQC2GYfAm zM%VMjIrMmJ@#*Oam)k?7>zm33aUXuC0J+U6TpXv_J=!D11&jU5&m15e#+~0W_*V%w zs^Ai7!1$ZWgoAOfzMlG(bx`x3N!yJKYO+tsKJjjNb#@hhQ4Lb+NQ)KbBl$oDd^w)g z^`TaOPEKr5v>(`sCogW4W$s@avsDZyEi`uBk<=lSY&dhP2{;R|v^~m9(PEG= zhP|A|3_j3D8<}$c`CE}ltEus(V~wsBWM2>F7+hX+8?l7L}U{5zL9AK_S6_KGTIU@$oL$g6_~6&v#}dUUv9JdJ1vfx883I`vNog z!h)FB)?73tZoOaZFI|j)D5v3~m2k~zWazA}^q3APQvkgZtSrr;Uk_aIN?WT^zaRJF zQ(t}BWSjpia4KF;$gY*RUbhhSKajr4l%3LxIi8JP++5ec!zDXrzSQ~uOm)%E&s+MM z-14%Y%Fg;Jq`yGcl>GzSAp(Xs=P~#Rd_ot;XpzjT0Y5dk=mwdzD0pc46XbugBH+nU z93Wyod5;eIcGyybtgaT#eCp+8?EO&&bE7Z!gBCOcjCo;%v4fK}jXAdiiJ*~rTVC4^ z#m;2NVJFW4m0f~Y&`@)^`fvXz5VhMzq}h^(9I_5#fi^rg3uyVQ5$PMcU4zkek{4-D zq(3!u`0A&H9@wgjjH^y+UED;=dVhYjj|(CLEP9mA`d(jQROSm+i$x5CZYVK}Dkvog z^{!4PWG^BLuXFk6Qwa}l#*NC;)w)_wT;xdRp?VXu_|CYeXJ$TY2a-u=a6vy!(JZw) z3s%-6=;>K4@`iEZIOdJ47^jJIcm|(1I%)VAWrDRt5>Lx`@7YxX@FoWWKDK?$trMBm zlg`Am1&5;p{0GBbNcUXt(c8Yt-q|@-K-uQ}(T*g>y{5n@SW5q;i|3D!KGC_Nye?j>*IX}a4dqN8H zP-VsJoKf-{1p#NDalPSW0&A+=e-1^z-Fr@#NrNiVj#Qxk#^9Vrw)Jm$Ii8RW%FgpI z-9Q4q)Yma2wwjtDCM*`OCp0vo&^s9cU1x};)KpV5v-nDa>c>YF{(rDLYA5bEmsfY9 zT~ChS{XI3Ux4tZVMb}&W?d^E!$q~v86v=RIe^K&Tn}5Hk@4i;+9!rT%(A=F`O=QSY zFtI2565=uGKSIruxA}kVvGy^TzAMAZcVVUuaRHT>?(QlTOwKdX1z z=6NI^q+Z?HhXgcj&@re1*Sc=u!;e@lL(NcKph35Hvt#VSIl*(s z@z~W1trHFmaS{9Sj(nmzt&H`eG@itShMKXljg(X$KgE zB(r1}|9;IA${a^rgKQGe{Tt(BPfGZ%+26wS;mpDoR-5`ep&S(lz)(-vQ20Ihr0HsY zrMcPXm(7|%z2-bLfcX8-TtJ!GOebup0FJD-|I)1+#4v`4XSlm-M*}MNhCc&0$4w_V z>}rkXB#~EB{nqaqlc&i_kdWKXd^D<^C{n&iL`g8Nwp#r=-+fCe{~x5i`-(g8IDgK5 zf3?XThQC33Y`ITVY|2TAp=p-jnkbrU)xH}kkhU19+C8ycoXP!*hXf#=}qmk` z#t1G%=O<9{gfBw(9RY^60vSh`&wBY7@jXj-HoT@Y_Vo3Keiux$CFJ)(qAwz*=H_uR zXwh@W$i=^)=DSUuCl8l+moi2L1BX{MIT9u8s#Y&kv}GhD0z4MV7E@zJz}Ij=d5qtg zjORyOq3QhpnMqI&CPe~3p>dnWPkhMQj_+ANLU^G)gcoAKw0OK}b*O4;BCSkdG%QWR z+Tm&}D&Y2K)ej2`%QNGI0Y-Azi$EYHydKa_R3OzgEnrala(D;D| z3!7IsGUbgBwA2|}aI;<@S_$iQH8-`o+$S^6&7#J}&ZcKGU#3n!Pt>QNy=(A#{YK&c zeq*p*V@41DYdgv_KC$Pr*lbK*>l0-!yq zn%dv2k8}?j-$5X9DtwZSr}F?_i$U2~e%e5T{2#q6(y0p=`?D<_lPpLj8>>t~pN3xv z;!61q2E=kO3*{2(rvM9TLtN*z1fM4ArUa%OwD5kIzT61S>cb1;sS9ks#8;hosNE>2 zZ?|eKmsr=QT2Xw`Ik#_XeuX913{nAKuC-=8j})l2PX?;a$RZn<$oS;3u+_CaUgJnoT6`#`1(`DgH-dj~3NWH*tC zwyd18MOW_k7Qr}u?5bsy7}wM}zZ-OT+oO3zcqbRYGKLc^e|NCBUh%-JV8xjY%Jb#4r`BN)A1DD-TaSEbVw5%q zpQUl0eqxbTs=kCbzbU=@V!|No(H0-5sD2{zmmK+1-iG+H)rc^`8?a24 zAm;F9T0ed0@}$T3^quP0F!Q$l&Pqc{=C_0uJn*m~b(@i8tOVmyri;3yJpeH~ zAYs^M(kfJ>3(JXr2FHKsZJgOJEGo?}v@)Bd=Ws~}Sfv)ImS7w8`iOwq@kptJTcV=m zXboS9Lfq|RV*(=89I!HL8`wb;M@?ykGGJUwEbR z66Tg3k^QBhaoAt8_&BhAQne%|HJv!5NvWr+R)!^rHchR|Y@S5gJE}yD*Fl#WDPs&R8!kE~{TA=P>7(yw2t|6zW6=MA33n=(bcD*yl*gyy8aX9=IZ)@84`y@fRMaJ0!oeez)uGCV%Smy0Q0?Cafm9rMr*6{M*9=n_e0ESYnp$xnXlr*BTB8 zPW4HB8&mOZVOX0Bv*b#6*jLys93{_-5BLQ=Db($=GTG`1i6&QXGyDx_iVqHGq&)i%zqa#Hg+z7D0nEn1hYHNrqibFC3{ z>U^E}nm%(dA-XzU%5at@^@IB#Atjfy@(!<`tuQ}Vnd$H~Q=e(_k>p>~s*{}qJkn=M z(p7xzuR96!C4}lVYNOlAdVH>MtzDiH*4)-VqqqG{xSqN5{T3?z>snAaOGDS^kNMdp zV(kTwWnSQTzf11^_695bRu__YRHKp<>}wCLvVNktfd2+x{I7k~{H09e(bb{wig3?Kt7z#&2gLR2oxFx3F<7}PEt3Vf+*Bf!plsh zFyobx8#OO;P7v8WttW$h+k_X6czGXQOsVHqWcM3CHp{r^$mUw(xxY2MsVxn@%)%Sj zb4%Xl0Bh6k1bAXU(_Q%m1&PvAURv!{o~dheQ#rHf1Vat|@Mht394Fb?KHp=R!SSf7P>DQd!TDnXs!MhCjniAh=jNL&}xlU&YW}z!~ zWz&gr;T8ZUn*Ls#{B2Uz(!OhTtomZEI|QFaXHVuBznKv1jjz;0t!p~2ViQM%buW)?1Q<`yvJoz(w6ql?OQ|%?n&jjGhwkC6=2NP$go``@I)1R1y%f z`#wWGt2JAs$rz2OKpR)%eWL$e-~{Dnv@5m2ev&1raLz>K!ay3(6m^@n>#nA}$YMvdkuf_Gg>UKq z(C*6otx@h%n*M=7qrith7l`1-t=yaZt>WqY*CNO@wQ_mkYFjG430S+Xx}ucy=H3Kp z48u04!MGDMfDnI-WO|Idvx=4v>&dwpg0AvpH15#m6JYZyj@~I0?dY)MEUgVR9CujZ zICn+w%-Ts|zq99tIx>P^I2(C$@Et!+%)Sy|AOm!Z6~X?Q>elsZT={nhQK)bzb#_f+ z*DaP#vtcwQQP=7G5K#nZL1&2xf`bQZ!Aj!BjZI82lk??uZ(3%Z1_EwQ@k0sbh`N!D ztW6^J0<4UKjY~qnwTSad)XG!*=4`2!M5}A5WwP6aVkKl<&?YhV17R+@HL4tVv3ih{JXJggeLWS@J#6d9M}vvL8|&x!+(dp}PbXAkK4)rK_*nYme1#M_9v#Cv!M!n1 zlbt&Yc1Y}!jpc>TM8C(7G`O2AkpWd9^|LOiHtFT6s;h=9hAeoiEs_wY);YEFC7SA4 z@_7oBE#Ubko4h^d97)D_GL>T#H@I9xHLZp9y?_&+Zv$?h8yMod60qKPb_?XLJ}JAIY^18f@8r_hGv{@k;J2*VP>i52MbU z8q=FX8C3Y~^FAl5d8)fC-mh3s=+mN#k8fW*ZCIC~Lww(XecG5IzUoLBWp6a^8};Dt zcbV<^&iL}Sh&}sVZ76zU66}SE$K?q3@$!}+YaB;5x7Scz`L8lJP?s))Sukx=rNAXo z{HDXtvX~tEX%+&-tYlBy*pCv!EZ=3q~7?+$p^IftlwMd-1m;B3u9IUIX{?BsST?V+Lb1s5xX#vUwJW zgM*XM^dYX~%pfH-HD5tNi4gTXw7p;x@1cw7Ro5*%2MS)Jeh;;m5ebX)>XXro3nKg1 z;e<=Ft<|x1gFf7Bu{y8Q;#(>gaTQc8ouTFd^(i_pdd&0ir66NId%74GI3(nL-xNK# zu-UOhe$(h=ctRQtSeJpqjC$p)1H+&W;D1H_UOq6?9P}tgWO!bcf516Y~;gb406X(?+mWB#P*^# z{;O(bxGH0A^Z32ONloUf6nm4AO#N}sGiQ5h;L|;Swl5jxtgC{{V5}um^+75!#=7k6 zGA-fKoy^M~4%=3Omh76E%3W-6u1en=^O4Bcx?PygPiAYQCFA+faVs=VWE%(#zYHGoaYqKZywX=wiAmMy>2UCJmaEa(2J|c_@>YDvRS?# zbLRT2HXQw&%8Gh$dEa(~Wbb*DTuJnTYlC8AU6lXBzWK!47koQ`!*}0|@)L6p- zz^r|j719d;&Hq+SKcDm;`jf}M<3n}tZUX0K9Uh|+vYMb8r)<^~BToy3fk!5i!44!| z@<<5^)MGW+HVS_agbF1{WPO72LEXDFv>c~P_(r_KoAl;@F*?ywX^|K|e|i6_Uzgp2 z2f4I2jS)=c@kZ7>06GZQK4wpqNsiWwt!8jS&KFHs_sW@)m-a;rHzLI z$m9;JX;bgiPoq|fj9xZGw76q*&p(r_6(-=uyV7L*ys?p&oTMjz`u!B;ZYn_o=aH$- zydnnI5pG2)ETeG@X7fq(%%9uXVS(q}O>ijov;SPMiHH)1~|&nU#LSev@?^ z^pVKs4~4j{U8=_fplQu;_NR-mS%$LW;z@NGHNUyBS}7oPFXyVWE4E`#>TcmRaH$&L zEN^hRez}s<$qsXds8k-9Pi+(W7I%E)Ytv)ujGS2V`}d-?=2=xSm5{^5^k%$mqe

x{Cm7#;*g+}H z%Ie&Gf)=I=CXr_A{5xjU6FcOxN2b(zVB^dC;MD>hmWHnui^;|2wmUP%AJbo;)5{e_ z!UOh_hKeX(nORmPCe-lj$wYxbI56IJY&9F9;l)ec!N{|lJ|}1^4P=qqx^rS9NR*q% z;GM+YAcgw+F8^}*FCV&0n>D@2C&aj|M<;=-B}{kOw!wLSMEoGI5v`DwD{;(j4DX4^ zr6FS{u7Ni>jonZLF|w&$=6#Ddg5*xfMq%%VN)y?~Kir7S`rxr^SAtnxlrjIzXm>M> z-bE^3U%dkSxeUe{8_il}+_*Be(N4h5w0$Gx%38E08niDH1l3g?#x>?QIq%$o9#@-w ztM%_#stb#GKj?{RbaxA>!YCqx(lHt5J=*F&vAmUwVD79OPODE7Eb`dJ-NkJmG6Ft7 zKI$G3V{}I4rQXazJs!BT#?vaS7vCGr}}<%hEyz=2Q+DXA%CnhG|I4%WOFVR}q7Lb)Wt=_<<*{?NKm!){7^2 zFWB+?=J2j{1&tZvMl&lH`5T&SiR~42x;BYwB{;v)ek+3(ipWN{hO`BT7hrpft zox$^2EBvxH!rF+)90-{Oqi6!i{?*T`)EC5MDAf`O;??D&qoMt%`Eqk}LpSuz(P|Td z3`CLRk1RbI;Z~Qf9FE~R5(ZcN^KX*t2z@hz0xOV+9@0!$th6UDp`Kv0Nr%d!xn+cc zojY@n-=Bib2scDK!E__Y=CyVgH9Gfe1%f~>neqyM-&bOI(?W|R-8wRna_xHSzT8V| z-i&CRXEvi*!m?1O2~Rb)2Ud$)X=n3sBU`t|0p&QLV=Qqz@5^|w?#(#WGT_>IO{Qp= z4jy2`{%zJ)U0@{;JK1vm#`%VM`_gDTgpM%P$H8zMSFX{*>AVj;DuZYED1|*kFkRYM zhRy1LeQ0cinx7{5^%WAqmg}@82wY5Sjm>3J&FrC&H4!*7SxYOUGFyogNAQHZA@$z> z02i9H%wf0jyF#z^UxBPfi>sxKBestlyg&ElvX7#Z?d-wn!fmq39#SsXoA_*D_M;pc z%M*9LU2=u%(e!^;O;=0Mt|#+Jh|2F(TJ1c&Ck)Wzj&YE(W?oS)$J>HThZ2|)u9L!i z@42bv&m~JyVO_4Cd@0;|kI9^-c~^7#Fq8lOFwy>L2~>^)1O*2xg=sIEW=)dQ^tON1 z!=O5T;@ox|5XOUA&`i_uYQ~WK7h|6qMQR_wj1UTLJyoxkuBf=i&C-eKtvL(Vba{o1 zsGps#Aj6fJp3;vZv-~I_2L|$WY@IV$LWM>HU2C?3gM)3xdRQb@hxrlxF8!{taY#qq zlJ-Ufdpy1g$xPgOwYlkN-npS@)DDfbv`JalS9P_wa`MNI@?$0T^oUuv?0Xed(jOqL zylW1O)DteYz2E_xug%&e$%}C@aeSjTQWjEL8K57QAL(WM*#tsM~D^2Gxop|E{4R>bGjDddEug$dG*wU0iimwX)zFIjq$xumZ6#Fc7 zM!a+|-H>)@zNrUlqa(wkHO6V)zy1X{9;)Fwl&Jif=~ z{P)tQBF*Pz%C?qWEaE_B65+9) zwA-TdZR{n_K0h{EGjLL)+G3#Qqq5(Z=U&PFOfY8Z;!V<&vxVGFX@kxEIV4W*BCh$m zHHq~?K(f?WqZMx?GoHr9qIkfr%(TX2{>0bma+|U>q>#zV7_IJIJ&Hs?JC9BG({V`V z#a30ZbWItL*vA7$6$YqrjT17yV!=#_R4EEZ#4T;XFdfyL=75Sz2OElC@*B9OmzEhD zSi)Of6AC=0ehl*gq1c=!Ykd!*%lMv&g+$E(d78=_T6_b_rPJ{%YP5G4$pD5D0-Kph zhR`pSeD;H}S;>;_3hqk$-&hEcXFnU$2hQBYL_G8}KB)XBqs`|VREt!Ktiw(2`lNqNq((^ba`a{Il{(h4-G2JDqkAFWCLW zVXf7s$IyM#*&WAh?R_ZH&TQ1L51!?1H{4R#Q}?LIN@$mgmLB(bxIQ`(28%ieT?9+< z_J{bpa4Q3bnC%Lq@cJTmimpW{g9ivM1|LfvY8_@Naz}}b=U#iq@cHAJ!v_r#x&qPu zC83~G=qM(%HzEvwobTuRGBMc<^=@)eA6;BTkDGrNA>$PcgwzQ{1VTP)`@vOnk;_L( zV9CN4^57OEh^x|gSTAVXEzAk zKZ*Go5ccb&&1-=(`UZ|D@iW=e6|>8X!wI{?8XLdRO-!JCM0s32hQcH_#;hIhP>0Wn zjjoWGsJ4DUChF0L$eFCIOg2+VcVmgQz6UqoBfa>m=S53+Hy zZAkdD-QEwk@fK$hLgHMuzMB^=>!iLjIVv^ZJ zf;M06tOAaNEqpt7XL!-%YbiX=8B5~L>FXnWR!V(j z9oX|%pW!i#&4DaAD|75<<^}a!#rLXt?0kR=b9YD-qNf(>x6R%?cxESN*%=?NfTiqL z9v7ruhjX#`Ff`MtH;qKuz5K*W6ua@euYYb%`%-=DUBB{sR2(eu&UjwEuXB`BHS?9s z>Qdmg4Xgwds;{{*YQx#jN9u$`nXg7PzqYyaw|;01Xk4@p^uFm}nqsUr1)6By4|w_u zxUeSG%ZwPP;B`LM$)+aebpI%9bDDt;osb!{D7=yLDV1%of4v@-)5Y*W--5?Hd$|St zXhl1f(yBk^;Vi)DGs}E_dBH?SS28w|I2RK2Hg8#JO3SAD$#Ov*8l(!*u3|CWhis)^ z<+-7Uizfe4c8zvwkSyihA)WWr9oM!vLMrwiF+u|~zK*~Tbm^d#zn`b|H7qhkrFz#K z_b)BIduyiq*>d9TRarI5FPD{s-@ezMAlKo$TMN-dOHtSGZ}5~V)xlxL^tzN7Iun7U$2Vnt4CzKvn@PXNnj{1Jx=g+@G5CO z4VImVJTFXXN3bb%G8@Y(jBC!>18YQsVI(3Gux+2%iZG*dX)APs$>2EPKwb3EiJIP2 zX9Z!cR+cMzpn}}Jf^u+Oc}+&Aj5Rr@vtvKxLE`YL6~=T}&RG`aSW$DHfW+lxD%FCr zc}7rSLDJv$tnDk>I)oO8(xvtn4nh9J$Am?SVPWj}Vey;&aT_5Z4*C*uWk7$1MXUAk z>le;wlW>FGSa=3dJ8kf&ba=zK-m2g|W$R+PUo$F{d{bco*YNkP+X_Jd!=aV3X^Xxl^#*LiGhZ@a)M)NYfi+})Drh| z?BFpd1x!N_2)4e^XH9|YkLye|lM?8=bT zmO8C8>UloofpxR469fHU>BDmWwYb^5etx;z&^mY24VF0gY#h6568dGS(WotQTeIM0 zI;JSTr{T{V;@X;Ip7y3XzCMt~grCgIJ}q-_vTVbLQM`EnG28-`NwpSlig5*G9*>3# zwe*~-aespKaH+bKF@H<5;(FzUnVV^#;Fa2)8vPps`0X@^*qgD31wpj~7wd#w5F>CO za8E#bHV-xfJ?cHMT?ovBF|->n?qKB2ug`J_j8%q(*|HZlR>UL=sQX1mb5Li#pQI_@ zED3wSbPniwrMcR)aM-V={QWD^r6k;RR=JjMO4A|?hgV()e+CD348+SKoW^4KtDpCl z6;7p7wyAQYM`BP*>Mx^*rR5$OY+doSn|KS61Xv$cPnT2QgATpWRa>?Q5+sz@O~`xw zu7t(sei}Mw-9F+j&QMPBoZ?W3@;7jvG=G`*VIf`M`E-G3qGXkeg7aKV1rAW8 z6avNFr8oq44(usqF6TEk6HixZEQL+glaq zvQoh?!M&KBy|*cfH~}8#xeWiLu;O~fj7$m`x8@{U;8Fyjr}~ba`+Rl&A0n3Ken7c} z^Y)r47;#}I-b_LQK>^Gu3IbtE4rq|o?X7=oN{uZob+vP)v{>lc6r@-|2{B9;X!yXV zSkpbcY-orh%KGjdk@g!GK$PqYe}<&4(3a8D8lFVS6I+W-|548Yj$}g7OLocOQulKP zvv8VVm^%eBMI^=6@v%agM#cHXMJozW5`d1>^`}=`-0I{fwQNT6=;`@1xLZvc^`)ZiU;i+lZV-3-b#QUzmyQfcp4Q%}yZuvWw*4v_uXdJ~ zT6IX1?Y`MBiJAS}PtmEB;(dOrm&#JoHfG7G;S~dV(mK6>;xYdCT2589ar{A;C&m69 zvMODZQqGYzxu&atzo4i2VgctIcR(^g>w1>(Nwow}k>MT}x0A!FNk38^AiNI!z{u+p zVP6P#iOx0b$~BHt*p$ME8DCM2rjl896tBXvaG~RC<3o-wnKoYxId6@TN;N2> zL(xJHtjFm1Sl4qfJCm7}^E(#ZM{cp7(`FV{%XnxgSg8R#Nt?)|sad~-Se@>HGr*mX zE;&T^%(|+KtuqlJ82*Vpi(#5F<|}{a?)hpXzhJ}a_n%o)eZu^D^DbZy7~VZgp%-MR zT<%+(?)GU4nQ2OS{Ta z_g{FS_ECH7C$JP@N8W)D3)Y`9H1K5W|A~2Cn&Pak&`qI`RG9~~!0YBj=QBtrN*vBs ze89ly8upM%=sp}|4Oo@a!=-AWYBM9%>E?**99q0XAwlx1+=8+J0Y_6-mb4tCqyrG$ zB<5bR_#vLwNQ_*fk!oPT*6*HN?Pj(*J>ZX$Yh;72s=z^1?yfy!ZgZjQVBV_FSWWII7^}Awh9}acwFF8p3!}dG z!-8m@SAMgD*mF_CRF&zFDtjiQxH{VgotB+cT{w)>^LQ@L#|nrOwAbQuU#mJeFse%VZqM3ogiIB8 z0qhBZ2#N0k5iHZcZ>o!YLYrZ*{*vF|_Cv_preZ3U&!^&A=gNd0S2Bw3NB(b|LIyH+ zD*5*^$l_S+@$m^ERD0XoM;)i6_y~z2SoNURRbd_hrNoZ01)uEarM0Hc>qqc!zhlgt zCFfEk3_kbJ7ZdV;oa_lF7H-6tdqPkDF|(r8d(J;iL_6wAe^tv0j3AFpPuJ*rygPdW zuao2lV+$yzoaW~nkbB}2dVS`F#zH;XYVT_X0TK-G+@ICwwhUG$ICv98ci&BoW3;&T z=w-0HEchndl*{AA>~YRsAm5atbLD5{*7)N02Hn2;EBE^QU+E+HGlEL42Zb)~k&TzP zzLomfv0>*4F5%}&2b&^@nN0!z>MIDgRvP0W*3>-12oD>oy{V3rES(#FPEWm)F)qeA zUAfPQUc;pVW~H|_ahvHk;5sMg)&y4>EaThT>3#zvZZ%J31sW9)ejV{`8R zgk|!GMX=vZvHTaLsRjaxI-MA>-w+i|Rd`%Tayg#r_AM4PIXmQg=e1+qc0F;LyO0>$ zY*7T5s7kS$SYkP;81p=r)8%+umzMo1rQ6Y(Rb+Q*b#!6lyqN18 zU-b!N9lr3(eb?9Wcnhq7;yNS*%*MK9w+1tMdhSA)GB{&~(qhQy`49ri0?Ue`gO>e{ z?tCeAT}mxkEI>i&j%zDaU4n7tB-H4&+RtM-b*LaT77)_hdDH!&R(p5u{yp$6@#O~Blhi_wV# zo`(e+f%An!fn#J^dae|5c%+Fc2L!*M=jtV^+`_70zFL*GX-$5rDSz<{&+BRM$DGCb+8K@31-{5H@U*hL&3 z*fque=i_RHm=V{#A2DW*(%l%&p-UL%CK~Zv#5J*~PJc@FzX@=n+6tKb++U6MzkK!! zHD3?7dS6^e%l3G`mqEOA z-L8^9<#iW`M&f(!BFkrM$TmSBJG81Hd()VctLt*bnR9pAhCh`u2HhzS@D*Z2Z6|K@ z;i>XWrgeyX+-<$dayk+ceh^gB3;5QiNh175Ly7S9gHc&ELeDL`d0+4Pj`2N-Wk|G-W3=u^lk-6lC|V7I)%j{dZE%#x5lWSnaegjEjYsAif&AWY z^&ZCWEu4B45fj55(F6=i6#+JKzmS=u;mRejsh^yensVz#%*qe%q8ayVR9VFImJ zpp$!1k+5r&3Tr}^WhY!zj<$rznj6zyjxx_j=t3c%0`7{C6kTV(p&Mycdlz`mKTAZ( zI&$WW(bM(b+Mv~i33E1zr5>x1ZgAW$2V`Tgd!ruEPwjGTNA-MVo+{`@{NdwdX4_}@ zsa#ViK=|OVZzrKZ`&!~2~Jv+GAOJ6mA&@OPi_9c03F31ZeV9Wj|04%fJWFE!y{TNH&y+ZQ&*_nCVzNFm|*83Zu{34~{`^3z$8bC}* zVM~|)705Hgl3D{4_#IUSti9QJY^E5TZNo5+**Gv{F<%L*x>(8_#M)B!p|Kb1Az=%` zzZLqUOZXD^>gUP#p>q?f{ec@%?-s3W)vzCa8WGd%U^PGdi?x8uYkOjfk^<$RZvuvO zPhay+Ns6uBEs&(`SQe3QwBz7|6)NeN8^)K-U@p`aKP?ep9Qm0Xc&Na^*65iq&Sj3P zBxE5uCyr%=2T5ro2Y1!{IMFK_<5*`$prz`(?e!rD?4sZ%&$fKG?OW;W|L1ZfVX!La z`5$q9TO=E^*Zj2$RSUm3)Ee8#w97J0;ms}J=d(J}!L4F{c#L8T+M&6p#IKs=Uym6t zn;+oZBYOA$1+-P@HoftW)}3LuaWtP0;{?wMrxGC^i}2&DuFj0N=VJ-QPF7qi<6#7@ zURdayMK^7ke@yKyWsFeH-O9f7eY4Mc8E$3&{B>+E864IuQWL$e!4|LBz9#L@enuW- zXPo`8w#dUFdBiRancUo&mW$DbL?)%*L}N}1MwiB|PqktyZZ zdgB$m+Hczo&meJ8V{96T#)so^xsUv)G*a%u=!HP#+E=+o+a-nB9&sXm12HQvuIow~ zf!3gb6EZTY#9!$=iHT}>+|ER4G4f>dPon{xefo+wv7^2j83Tb?lte5-Jwv6_oUMZU zy+NZJVI1u&lh0WAS&73KgYrfr>l2D7fYv}8;>$sS{=qsF>o4-2&VZ;Yj2A_z{F7Qg z+lOtA&iR(KBg*?^^}mzrY}GtZA7aC4t$sTW8nUI(1CTZZF8Qp&ymlV6nFMT5|78)b z(@;9s=*RMZ<0^A^g-8h%E1s zy{)|K?tUUDGL~0>ot&&Jo{p-xGT8C7 zv&;MRJi!~2{XHd6I3T-;glMd5H=#Yl^v}t9(H=WbMegMff|RjdkDSd&gQ$kUHdoTz zinZT+hPIR(hsd;qt&;`nzvjg^rAfFG&yy;*OaDkEO$29(Hc3n!yI*cbvDErML(=6f z)IsK!iz2(Dk2vI=8`+V;nhq9H8??Tv7eIlBfm1k+iF5)VV3LkxEl_<}f^dI&9C3%2*DX4h2NvwJ%2F)gZ9hA5@(w$nz$5l4~$-i$7P+2Egd02 z(SoN=`p}P_r?edJ;FfN`RzgWZakntO`{U3~!)?^}ZbPRQgH!~_X9!|1`|SVsm1auJ zB45vpU7Bp(@SgWDu05iyr3Mn9Z{QPzlfk00L*HyvdM$RB*_Q#!-g#hf>k%&0-o0kG z@0KT3ZGvJlG;kVfHW;0e_M}rNL6M-xuG4M=gM+Q8s!5V_awZQC%ffo2BlGE+6@7TL zb>%N6MDG^h`AJ3CFah!56TWoTzk8dJ*L>l#Rp&74EdY}*&ung#`PgF0s!8Lz{00qtmsPpTwcGgJiHl$SiK7uEG z959Jv5(jMs6oQ(6^!ysW=NP56j;&+g0-=CCHULH^4I)9Jq`FMPGQQCgUr{3W1=Q^D z_`GOi&lVHS)l+h;xgzE=cwK0qQ0TlZH}4@8Of0I{GUNz(d3i8=+RgD&6l6Mjs8H3p zJ=$VL0md~5dSg6qN>G=q zJuV_8B~=m^hu7AARK~{n1f#(jUp9QgC5)3fZg1DXM45c~aCJ>AQqh(}pk%zH;)6M7 zmE}lW)bhau4r`o14Y_A%5$mPql4ZNaeI2>}N|p@WwY)TYG7rew@T&B|qd#Y^^Tf;z zzl!;u)Vc0Lrzsu}0SU=vFWZCNbzhlx>Axi4%2=Me#Y9DIT|uW|#J@K{BPWmZ5vyq4 z`~H8gZE(gFcS6W?@t)xlYa&GyIY|a>INh!+jw_Y9ck_X|;UA^D3)4S|DeGmAM`p7m z?X3nn#5bhDl}ygDCDXHYnaTg1xzi3BYs=A5Q5)1{Tr<0Ed`2c`MS}-{GWb21VcwO1 zbqD_!im)iv`y+RL7)niDU0qdOJ^T|~6@+Y0nY&bfLfBp;%}V)Q+k$9)09-_10q;&_ zVNnU1x60pw0+r?T7)S+xG+AC!AOF;hDZe`@Q0ZF<1djfb7B*6W_;gF->v#mZd~dnd z7-bRSOfkb9guklJ?~>Vd)Do!GeB8K{F67s$7nUqdm3T_O!VcgeZE0y?(!0~Ft*cW3 z;ebGF+Mp?i^qwnvjxf$zEw9VzkgpLqNJi$S02y|Z(zcCFcmHuvLR+r}Ss za0+vG8*zi|ev4>WrbyE(35q_}zwDW4t8U01%eYmi9N(cefLK}V)y~HX!e!bOT-~BPDaTwBYu3fzK=kBbE9Y$a(d;Rgwc?Y=;lp(F$#7QH#&lEfy=mkR^$ZJ#4QAZ6cwD zsQlJGPgfw{C|+n2=t>7|EM^ul*)E$zSAGJ7Z7kLkw6ZN%{wjhMJQa>1-DBZy1hT`G zFZ2IWjp!D`)qdM!E%draLtrCVUr_*-r?D9C%*f#Qy1BPU`!y9dT}dY#NgErJ!)Sb_ zhqEk}_BST%B`~&|3D!uv7m8*~ZYT%qcDtmY{)c2^JWo#CluE2py`SoB!#u|&eFa<_ zGwqy*1|gr)aX2hyqE~DoYgi%EB%mzr^gU!e;uy>q{0C42Fy~+8ct71jAI4 zQ?D3Al$AdLm(Ku2@)<#W8}W6rus{l>-4)#2xCpsz*RwrViy9j%L3E(y7-p<6NP@pU zAAN8rV1KU=*w=)bWf_IvhBZ4I5X14Z*5`zl=aQcve$3xxHtIvFYPb?IRA_{U2rf#9 zkMI7{v<$lk2A7+pNK;|aC$2=dgTeC+j7ijTsmfaOwURKCx~_DaVMb@LMxFx(6Zr0s z=H=HU`y7G~LQ&Y=EvoJIaF5m!f`@FaUNefIdj9uRZyAIU%%Bj41ZgT&Dxc6ID;QX; z7A87a|6cIYv6zZFw&b@@^L}5@r1!_KNR`NW?GZlLT??c-yrO8^o@?7V=2|?iD%fgW z^8Tb9KI1;bt0)0Usvv)KcRp^IV>chhhJ%AE6p@9A6?-Tuib44KN{!oo`M0JaaW`vp zs++8&71z5t>CEI|oHRpG-#ZcCXX5#a*x3rwwUZ@)*bk@y!3w(ijJossZ~NKGMG-F< zYMP{sMsCeeAu@!}ye5xxX$1xIgTFSgtpfJT%M_W&eaX4H_PioR$_skC0#WjS2d6_m55}Hpp!U8|Smj zk4?)R4)V@n!xCH-#x>gxrCgmIR$UDWQ8NaoIrRf-Pji zO<!D z)t<(vzQ`1_zSIBtmIfrMvNVD|_Ia>zi#}3JPfq}bdqyMRj5qEJ1z=-SeGUK5zfbnc znwoM>P9I)S_+Dw{e8=Fjj}BKH!a-TN0<>BRHZ#Uc!R-U*gxK8r>h;?yCnHPe!NTjGehz| z=h3@JI_G}TB91nMjE5)v?(T((mNqFdF=!gvZ&A15$Ya(}`W9B_BZU`@-Qb4Zq@j}& z+rYqpx_YvjuCAP)pCEG@VO(*EpouyNr)(w?1~Y}xsd=({LPA3^0RVAnX=xvc#n0Jq z@|U>iz?OL%K_X}PuJ-?&`v1KBf3JAYeD=x=4k_t1$~%D(%HPRIDoRv}83p|hzj2z2 literal 0 HcmV?d00001 diff --git a/docs/static/podman.png b/docs/static/podman.png new file mode 100644 index 0000000000000000000000000000000000000000..9d5d7a41f8bd0126e4ce1f09a83eeb423f1cc801 GIT binary patch literal 16436 zcmb_@gL5U_7i~Dn1QVN+nb=Mywr$(CZDV5Fwv%td$qgp9ZCkJNdw;>JdbjGPZ*{s) zcc0#8W39C#$_sk4xq?#Sr?DLl#?}A^Rk_ZjyY1 zX;wXNyykU{6&m$C%Cuk+`#na4Mj#LtT1HtRCtI)q_SKfJB&L+NY`hBUn&FNkLV#+6Bch|It z!q9}Mt^L4ag4DJOE}Dbe@SQ6$w?v)_{KD*323r3S6I*Rdr7JNiOEgf%j6FEGU z(rayJE&qM}ooD9k0!Glr@plJ@HwExkz8Ff02?JZ;^CZ;i2W-CDi)%OmFL2@W3ob~J z=??6Kah8w~fjNReghHlH5~Rog10x2L5Ef8&Up~urb5{|09_G8?d?+D@f~5`q>8uEw zR}dKPzv$qiB2P(Kps#_VoY8^TP$L)!PuFp-Nz4u|EHK!7f`qP?P|y?)2B(;$3t4oq z&fLpq%P;9^%g}M-K}hIhJCUBwY&PL-ww~^x_s^UKH%7E@>i@%L{~=a8Tsr6E)-xQW zyNo0R4Y$aX6QH;*iO{>eF_H5p{*qlW<<9MxEFZYqT)*lDj&vNz`n*B12XZ-&e=hYK zl}c*o6;IO%$)>{EPvEvnz{mI#UwjO1xVlc5^Z4p;2R#`nR8mH7UW2eDadzh)c|*$O ze+PhtM5iTXGsv;S0}t|)3~j6#Z4?T-<6o`I=5o|_C(({rk6Pj~-X@ZyiGZ#pd6%8_ z`?lH^7kF$XPEjAyEXi)B2l`*fZ~8Vk0-L&rtmHIl~!eF;6f}1 zz261TUOQfF+HmG#N=6vC0Xl#G$HA62>N325Ty5R?H6ctyhqrkuW5fB8i)3*KeahWj zC8KGtW77g)au0Uub=1olWBG`?;L(E|_6ANLFML55zj1$GsR_T~JT5UrOAVO~63vc_ z07G-r5Kp>-4vjlzE%Q+R@Sw4il`MR3pXWy+#)O5`%q&?_7skzYh}m=g6$b_rAk-Jv zXR1$NKZ`b*B;BxuwQ|qbN38FWe~rgLY9=0G%LL5p&Uf&1)JBHB1q$^Dk}rc%Xilg^ zH6lzh{t9fFHf1wMr*QfiC2(KAIB6pMj<6Rp#^!uyW?;umTHI}ECr0()}V*0j==wJKpQ$FD{PM+J7%D#*iajbeCM^; z`KK+AR6{v#C(Dm6Ic>_j94MG z@J+95wVQ#=MFyjM{ySV!WPa$ZJoh;h!VV}rJp6KOOgf#xAaHm9GEb#amHYj1XDpjb zZMnq;&lG#;ps#MZgq@N6ibq}!k~iFwz4_);e$5$JS(x#fm)f&dC@G5YU}&>xUo|B3 z<~HvW`cAsN9@vihdh8{$>C-V(v{=ZD$Vw|*vi*bWv`OGC zubDD$@}G_09dRZCHFYBf`;TR(W#3-luKIR|cX^MqE-)AkA@naSw5}8V8y*fjnkfaj zEa-%VcLjT?sl=cNOO#Xp^-2go_}MCwXHeE02{v3 z<%EoqlF|@+D8x&=Uw<@-a=K6qi4kE3k+^{ENE(=C2~ZLr#BlOu@3TDAVb5WpvmU ze!e{t0*s)2D-_TDs5tfV>dN4H_b*#2XbuTmbLEzN?`Jo?Alzm()*S2S3zKvwv$UoD z&mH~}f=qTq_OvwF&4k;HHJ^Y#Ptl@{8a;kbhU7{blVt^QTN zk2mkj|6p7^^APda7z_ts8yXt0*{rdDlgSjQ)#@6XnmTUyJUA?wq)q{|3K(mzLRfsy z4m%ll%q6C$f==PoYn-A9%=7gi{fbkR4t#(r@rCqAY%XKMK*p{^7_gjgH-nHME_dSi z8Z76i6crWQj!H6R^;oPI8B8a#B%Ga{vk57Y-1^bDtW>Dj5M8>N-cSlkRUr~Rq5|lWclXJ!>3u>S5*92O9~c$dFydz!S8$SR zFxD&xg$s{WkFU5U#Mgc(C@60mo+s5a=h%);PG7%$>)Ax|GVO6&_b9bGn8?=a(5|UG zNiwQ?#5d$|_QGX`FV4^NE+Ai7%fGW~CiM(IX>lPcd0nRk{Kb3;=Ikbp&bj zc>5w=ajX`l>B6a%OU4cgf+16 zExfxHaXsb?jKyv}OSM{)^Wkh|0ZF{pDhG{??46dDt&zIyzhb&(bJZ}7w_$5MrK-#{ zAf)D^#P9m*EzWM~*g+UTTpZY~K1ti4vCA~tP3G`Sf{dj02{S@jwi4SFXf*C|{Jgd` zzOZ#-upTI$f|HHY!ByNr?p;tCNa)s;)pFHHQnQC(S^ag;d}yfU6t&>m$j+mpouXkIBOssW1Wb- zvr&Dh87=~{!yhUT4(eJ!7J`qexB=zl#8m;jlFoVH@27A2Aoiq+0tpmAEmOfl1cFO# zwh0no8h1kDDb(1J?XUiJ$zRJZ9U6!4-|N6twnO#kQH&#~5NQsA0ue9Y<~& z^ZG8BOv~tSxFHhlfBjJ*TT(jd*Hz#Z_Kje95fl|b;`sHS2&E?EyZ_hSKzxW~6OF3- ztE-ROWHnvNG9S*1a}vXEP#uBCW#|IL|MB-I9_PWrBr8@5&zgN34y45@IY-qBym2g6 z&N3*wPa(}GVWF#pq~H8$FCA>ur&??rGQO{QwQL3d&AkHQ>(~BLye}B8@kd8DRNSjP z?w9%-{$O5u#chJZzoFltrzKi5#-*LM?#x&Jq~+=>X+7*ufzCW1CZ7~{9l%3@}$fW z)F#3OqxB+Hm6bue+g4zvn19caFy|bb0b|14}29REX!d{^mcTvz8Sx0Pp6=~`;Cj7X2p4N5lp0t$; z4-goncXZK%{QELiEA`|0R_GKyJPnN$yIok$!Hg4H;jZffduC7L&pnINmgtu4-rEaK zk3?>RgAKxjVaxNlICkyFtbE^jgkB;?3ftDe(&xsGq3Ze=wEeP~4`M7iH3_)g>(u!D zeSVJgORBt%?bSiM{~{^q#{=RLAx|>5;XwF)iNzYEBYRoR>^ayq6UsXdVb8%@Qj*C; zN5o>C4?igK{oH;9azO3ehg7gnl@z15LG`0NN{zhC6&FGzTrmGPp zOuwoicQ9uaNu2CoF26ceZ0I2%XE~!#;RYl7bCoiz6x_>ZZH^s(J?&@K+H>d{9ZgRz zcDf;zSQOUP`DbO(VAAUj1VTG4HCr9f?^jko;nVQyakL)TZH3=gYL{M_pT!=ApP}<@ zPZ)+~B)jRDU5vg9HJ}l3aFZDZt zxK9_@5fxu=~E zeoAak3nfEY0?@`;wCpohzCC56Ec#7f8H{?TEA1rT2RD)Lp{}8W0^6fvmFv` z@z|;g{XQxsWZt;^K3@3TwA? zJ8ATKviRPPQd_Jjv9cI>j~459%9o_r<$Bs0f>Qqa>u5C8eW$CCi*k_WpZy4^unMyj zdqf*t=bSM-xV;x%G8OKf|H;Z`lplWgl|rr?Pl2RWKOJwz5MPcKzw2L>RUkYJJxi&J z3wL#Jtv?4+Qhn$5YZXoFXP#cy7jf}6uAw8A9+k-NE)~w-`6=%N4Y%w3$jQm^Fr4S( zC{1XfNlumz+O#cA6sOnJ<_v~VlG8X|c~E+Wc#v;*!M7$z8o0k3EE?Uty^F@;;~KQy zAYKx?z^*9F6MTS6stpYfyHTm}m$+tcEDCl*K1Laa^Xt4xH(VY83Zf0+gDa7e+SypVPwW3o}>?z;W^Ug9D~C;t8&6CAunj}w{r|H>cKBSLS^Q0&JH-_=!13dR7esiSlG`D(rX|AJt!nN4X3xLi%g zGguL@8T>_KaHoKZ=aOWqI<#^1aaybzURgKbn%6HA2kD=*#c zhSHUbk?E~gEy-)+#ESS&EsunbUq$=J;5x|ERj&3bzsNWdQLj+Hr|;}eg(D<`krr?t z@N2xlo$VbsmVbcBoaEnR;S=$lhU)$5*iwre8S`|H<_9T8eL*7Bw)rh1DH%yB7%zUd zhBs%eOiw>BjC7`lbTRp@I4a=Mlft!VjV|5nTUvA~z3}T9s0+809m4RtH&Wb}W7?NR)S}De;;1Fxvc9SXGF6jfKn< z=z+PeXq!i_$8TeI*JhBEye0#mx=ZdcC_8{bB0OcaoTcDfT{$?Rt`S=B2YY#AZU3(! zIzm$ANf*p9pqTh3;2=3MU9#M{c-JbggKJW{KOP4o{mxR#!QmZy^V%~@08vsB*L4{t z+2%H5s<^y9mdL+MF_Rjb<21lqx^xDI5L5O2yPa2_M4Z@)qq7nrxf`aTXC_MoVqa|t zR;SyCw1L5QpXavmw5O-1DxJ=hlbX(05MAe~Gp>Sy!t3KsocGlr)=x4rRCILVhjX6A z78~mBFkOiqbH8uZJOUOy@E`I-g4+gw3CrH!omFp@kQW;5N^!b`~8Wo}iQ z46G^~ z`EM-ekJ%;5Km1RWOdelZJa+1g;7BKjdgfPaP6_zGP%5~A5ma0S*4!#U&-jZ~vIa#- z_2NezPmp0;5(T%?oD#{bbDS=w`(+WPF0Crg%k&|mFQhv8f#6rTmBB`KlC>|<>b}j& zqWp!l_)<}6GS&F}!|iDI;=)DD!9h1Xys37RBo($&zy|thsbK&~SqkNOdmUY2P2s&w zp-{B@ag@4N`J{7VY;5&+8pkP)z9pyMI^?kmXnW_+pJ~=8Oj;_pyw6mRo{#TBOJr7f zv!LP&1K->2PIeyzlI!JPj0G4E63b@A2DRWF$tm$*d->D=ceP&KbSQ7I-YiIsZ^G#% zjdDwejWtcGl1rMcHLp6Yhx2LSAR!0N?9v_A-TjW<`tWe^J_3b0iQi=24{L7DhyMaf zWY(BZ<|@^kc1KW1NPO#E0mS5T&8<>Fatp(_uphCyex;j&imC|stTF|MgybhNIjnfK zE(94rP~mNO3k3k7~joU~mF@I)ewx?^ACEo)IhHDqGsTiI{@JPDib9n8b z51yR;6Mn$3JG6HV`+FuGpC#eZlI*3c$5IY5liWWY9qr``h>K&pLNka22h5F>%1B78 zkN0uSwfI02W_w&4|Mr@|wziR%%(=V2*U~2VuGwU+@U@PCvwee$%Pq7xqRb)LZD<}@ z4o=+Tj{Xmpq=csYzXW<+_pP1n>#>yL;-?_ms8ZpwSk8~gfr0!rg4l%xymN%vYo@`T zKSD@Tn8Yki+YC6aomk^qO+P8hP{0EO+Gsf34`JJf<&xgkhB_~LKr42=+0%Kki42X=fUSofHB-YxR7hyr@g$t zqHM*d)yg^7BH?*sO-YBL2aklmF}8tJ`c{nYmmN>wCnhuDAiaqFQEZoA-5NMqkNl$cBVJAOmUJ!NI6x;=fI#B8GBcko$LpOt7sRMk>>K<+$MOanpnwf@vc zl~vFQ$_wkER+@XFa-piI7OpGug+hJRe|f|gqzH*0>)}>fI{8wT&zvSCO>rsu2e+#jzu&L{IR%~lJJ0O)mjHB8j#a-s^rW*AINOpC2{L~^`O zKQq~Ewm1KSe0+YMt}uZQ#D%W6`v6z1(gvdR38yDZ>YqO(NxwSBvlT?+;76uNYR~ex)o6cr%R;w^qVqW-pWlJpvD*&@gF!a}0TY*!VnQ#i%YTY@g8Bc!Y>98Kfl<@(;k zR%1Z87GGB?|D^NPSjE8VI2||BS7wid3`uQGuRJ<}^Xo&F)t4&O(neEf0fYNkE`KCE z{+GwE${S;23%zCMh&VX#j@ooPC6+Zmvi_cIZF$+s$>}+lbp#QG8ND|hkJ*9-DnyG) zCqKYWZDwW;y44fPD>@d>=6CpgNgFr((a3R!=lfr{3^PRS!glT~CwnkyHI=bZrVOA) zUgp}mHY~u?>o_f_>xd4owM!R-xW5zY>guZQ3?0;loiDlhpKSk`wXV zsgyw~80%kzRq!YG+RGM5(3M?!^o;`4{sy^FTES7KfGEG7e|uFcL~rZga`&;4vC9HL zX48IoO4c?LA75VyMH^Z3gM(CV=;JUdx+olOcl(%DST6inZ8_zlm6(X<-qBhn?(TUN zOAX7*(imc9}N!q1sNNA1gO%bFV?#)mKrz#J1-D>AY`$?AA8vN=o>o4kCc*xg@gOqra-)9 zkS`qCO2)8ht;S!i*{pcS2&tfMazZ33goZ>fIV>p`tzSWBG9Fcrq^s zS6dvZqc$qMc4?FTGd7sqLM}&+W)U+Xqalk_Z!p&>|Y&09xLAnK7R_fdQ zr9j+;W8Y6}x5Tq1fQGoodM$5=Mzd*z)KI#1o2k&jNNY#$CJ3$EyTlf|>DoS} zcGh@bJx2m{S5HM7(=o-rOlcoh#f>>7ya^E=o?7kB$HyBPMVk)m<^GJ6iz^Hfj~(^! zK*~LBB?}#8DIzfu)zPs5C?g^Lo` zkFMC!|K#qXl{|fP_d+4Wj2vfqJnwAY-L)=gYim0-!EyJ>?Y08k!aUmy%gTb#%n`JH z>y$r~FmFy5r9pP7BDJFLM~x+>VOx3TB#+8`VEhQQvK`NVGw2j1yn3ln`(x=&sJkFb=PDu&{2TUpQuYox)r54Xr5 z5E2q9PK5jM@c7mB^c5!+BqE$S5B24Xyt?v0&s&)JLJh6Y%QyX#c_R0>JLymaoUzBN z9l1^sMw#DQcHPOPIi87_v>A*>qPQ`*&`xtIkmmTS;cH$GJC5zJa0q!a>eN}XKY1#kw_C zyiNxbJg|y`=Mq0I)!t9(cApQ1&wrKUZEf+@mRGE%xG-JbuAdRt7F;++b;BdmWq;O* z1cn+=9N829T5MKQUc^#CgOgtOJAn7ktW#aIIVZ_NjZ2};Fu?uN;~TK*c|t&=as5?} zFIyS_1tVf(zjJV4m(C34@N{al+9CqXK#|h{h3CULC?F{8E0UWX!FsnFpFuut{gO3{ z-F#BPw&6Dd-qF8Z-hx`q9F@`u2?-q8H7TiNoTlzDY}CI>ld*j6{^a3OcevxIQ9y@V zim$jOgP1-hcbss8NGVXM+Y4@`=^p#8=CDv59x7Msj7MWL7F1hkzKQHI z6)PjHnkC~x3J|^pa+}fJjL`Lb!de3Z24Q6b*)cOmv;YA9ziNRomK9bF^M$Pg*3!l` z0(^It#Ep{(w_1~}f8_*=Sy91(@dXLL#3ah}@#(qF>%upK%|=?^pHC=^tj+{P)?)J# z5Xbi(K0S>p9NgE(|L4!2h?tl{52sdQs4ve>x8`)?f*D!lB)seEO%`jB+uJwe)#nXV zk=Ng1Z{QF(-O{{K%j*5rQ*KegE!5!;VRBcw4Ra5;OpMMnbg3pUXh$wxo!jyHwlLUBkTMtZ07)c|tt}ISs-`+tTSOZ-)22eC zk&3gkhOJVn*){5wH7!tsz7!gi!Sr?DGEA&>FUd0^7a0!LgL5gC zDyKdsCUyWjTWLKQh(P-+h=89?ua6Gb*Vms>i~#`x0z#IPxk%$iRonB36qp!$CVC$?$# z;;?($!2%C}T*~c1LC1ZYTiVB>EMLo1?p6r=V=WJ%NEESLod{VhN(z&(_Zs4uSzh5d z<&dY#%b+y{uAOY#$yZxdQq3Z+AxcWX<8@M41x&+@k`aLie_x0t@zWDs9CuW>3MOO6 zA8KyXc~%4(xx0&pRVTGFlXQCUR`+=^)8Z2DI1}e>;VC3S8ja*atQAh8XU7M>HVF5c zl=gc>fb!+J8Drj8)AI=d?fn6PqJSt2Ou*&C!|;)F4(9LGRc#v3=l&j_7sq~?ptdTr z!F|)twaSJgc9{$)r^`!}KZ07S6eI_q^`p9an$yF6@qECzlwO5ZbEl4zxLY2>VyC@O zKcjwv7SUhAja#Q0J!>tILA_NEW<)6(zQXA^U4rts23bw!XLrl0!r;hNXX@8UE49&E z)#P-ec%i$;`ENS>;G7yO1PaaO(P9?*1H&5xW*S2Wvq3-p!ViXc`R&1mOId>uRV=}M z&?PPFbJwJVZMhtG!`n?jnCAd~P}6ZtMM6TtFn--?c(AuuZ!tqQU!w!s9*lb0j-nSA z7vFq;xuADTQNR!g0MFt`7xVQ6e}DINU$WUf$TnU|)w;U5E6L0A2WUuOZeU?yiGTit z;^X6^>wR@x0Y<5HYS$KQmTgm&PH_82>d#(AZ{?I%wT}F?BF;FdQ|wDJ2Q@)W`$TS1 zS>mXe`?eO5iORJ2XB6a(2S2G`Z#Xy-itVV9lvk#p%LX|{3a$QJ}&gWi+FneT4}X2TW+$zn82mV4B5@*n6Rqt`7>07c)H&G;@p)GSz|X> zfl;m1I<)7=8q;a?oS3INg>i16R^UpL%UY&5>?Nse=XOAW2>M7aGrdc&wqP8uxcug_ zE}^Y7#$B0+(@@~sqDx43316RHGn#bLRqYwtb}gUGou|C;)mVl+T5@^>R=^2-O#E7L zFyD7FhkNH<-UpTffNv{aerj(1S5}Is%0HTOZy2AolN7ZIi186YNZfhDRM za&>=idb(Ku3I9+km0aG=$o)=Kd=sh`i^Vqp4x-WNKnI9q;8%bAcOmn&j=LkAu-sfC z0QT5DIQX5KIt-ZbVuJ~gWpu>hywZW5tt2K<+qh(|urd0d$Cql1;Fk+RnGzAUBou#hs(KR46z?lE} z22`9u0+^~ax{|(?E@sqXdZ|BF)%Mk=6vmy0D2i=mWo1k3HhR!8F}Kgobg0pkB_zHs z=z7rt(d(14xjkL_EfDlY#Y0b36*G?K9-EYuG#ZaR7Kr&6e0;r@2UZ%Gxqk1QfV{}K zAJ0fThJ@aO&hwuhFHbLNd&TX|kALv;DXE%Eee119&bV(^b^W}gO!8EX&y=n?3;HcF zl(I1nuffWP;Kctf4kx7Sy~HoELz%uPzqgNi1eJVZX^KV!Fy)JD-i11xBB@}N+|Mml z)9+Z8-yi&NP|j<*JV14~w;u4v^dB1u{_3OZw&CyrymQip$p+X;wY~42hK7dNoQ^;7 z*lh;@Cj;=hApjaL_xP-8GXbEt^8r~1AFbq!WZI+*@x&&p#{|)XJ--?l} zi}gO5HCv{dp&F z|Maxt_05e&UEcOXtJ{Sxq1P$naWk0Hy?b^G+Ic!<`YRFx}@n4BrA{9())B_k%f;>dsmDIAQX z^6Ll6KW~4rdQT6^8)T#YBGAJ}7MF|pK3ga$aMgc5N1mZCr(0v7Vr06XjOc(qkRfA-{&zfA3FKYEA|fhQ`%YvbAtBc5T{xvP ze}liG886i69EVkNVY+tjv|shetSBEQwwJZd{b8-uR`eY8E(V*c!>7o(n~uoXzPd*iTQy0%<-`O~1_#$N`>*8^nS4ZQ)1CR^oGmx+ zgyPw5j%-1`zMR>mKipk4zQ6t0G4dVJ7%q}PTLM%Aau~dLIqpeU;l<#^KiqXe( z!9Oe2?PeTpL&>`-DT=-adxm8$Kt@Uxj6VCxZegJ@PE<^+Qi`q}c>83?7#M@A?M@CH z_)WHJ8KhDb7xQL+6G@w_mlAJpo%biQr2y)1lIH=Rea-pD3tQI#)n6{|m(^FfH>Z@v zNN_+i_*{5cu_@e1_nwJ+9i=8|s)oT|KK-rT>Nz)|!i>>SxjxjmtU~mZH#C1;7GR#1 zm-d=!aM+0}Dm2DW`WZ~su2?>5Dyl}A)-wR8xn=$iH`5~`+&E8dk{*_X^o9CEHaehL z#%s1r!B>B*G!l8Fw^m1;zpmR@#!~05UoX_|^ zo+fd#zo)ZUpvsn_)iM$3Fq@9)0~I!lvCsIy=IJe)QpxS(!pPXsMfsdKWj z`CRUv7MaXpN+cBxnsP@g`qh?Ie=wb>Qhp|`)L23uG8&d7%DBDx)g~I$rGpbsMk2rXT>bP*PFtM$xC#>Ho)lZ^z?b$-lMsXJBK4=xf7b+6%#OONZN7 zi`Bw}Y2(raPxy_Pot>RpukRb!;c7ecZsw_mYm$#1fe;2Z>$0dd_Lqu2{R_3>#l}+# zs!3uhs>px;%@(#~Ja-!4YCjnZjzxaV_z@^nY_I2#ZjWt!ZN zYG|#G9`)SRkyf{4NOh7_TnjfoPSjP+bzZ}Twd2V#=axxmS1}rB&ih-+ZT>w>lfU?rjn--s(!t4(kA#|MUnn2bl(#O0 zU-_1HyCY*x8a;}3?jBp6C&>UIWR!^p0jMOfhUw~fqua}EXEu6kWQdz>ay@2kC8`S& z8k_f@kG2-BTh01I+KUkHiRoz{l`H)Op6lVg{ZE-4i1ycOP>f6_+nu7M2YB*R#?f9* zVg{lMI8SfjH!9Ul8Bjwckdx?v^W_`YVvlN3T0m35wY7$_W84}D3xi$Glp8PQcTP#3h4KT|+}n9EOejs&-3xeHaiE(ym~ z_)B`GO=o4JIQ`HMPo7?99mya=p<^t8B*NPJZ;X*DAFGPZ^Z4XZQLJCH$Ww z_y9qE9XFQ?I&mq~z%6Sy{w)4FkvLbbIfPBjW#fxe)i~XlcMzk|5DMN(a(@^S8Zt7! zt}da0fkBmScO<1!&Yx3V@rCBpDJH(G%)-RWM(FADI9l0p{yG{EyxZau5x9b+OH@Pz z3fP|FZr3yOmEF3qd=%L8Lq~>GEYiSzZ$s1&2(|@AYeJ)AZn| z89Bl$E;o~qHg&O3LRR`MswckFJkWA8mXondkIHJNEwdKVO3GC3)1LXifPR4q=V20R zz$e>-c4K~2+TFHSpo`kJ!9Pt|lqQSS$-F1ZUQ(vF1f zQsKlW^0K0Z{KAj7{VY1;ppQog)aShJ?@So%WfT4HNlgW!m+mzr8&C zFtmT#>wcni|Di(!M}A&G@k0ddl8PPxs2N_}*GOip zVb0~+nSIXFdG!?V;IK!1{eYR9?iWIIJzEny(e}o^Ck?Qm>OllH9vFQ7bu~THSn4~D zqqNOm#n~K;1gXlt&Q7J9u>NhPvHDjO6Y)`f6ccCL@c;k>>=1l&Mm)Y+Wv@d==Wh@$ z!J*vJoV+Vk|3yho#$^rWY*=-xa6qPaIV{!#i-i_qN~zPt)fkcwOq+8*=m;(&X_v@m zyZV~Vq$*K6q3~Fm74Z}uU0ZpZib5w>Z`;yQXR#qSQ~$JHn>{X_z3cmMVN?3^!`LzsZos22&pmF!~krQdGiwNb85c+hrP)d)9lVH zu$jK(Wo{yM9t=kxk!3iNe>*XHY@d8;0*FaK3RKHYvlyG0$k1+()!na|dFFp6#YUG( zyyorRPsizpY({OiChBQg&@jQpTK7V;I$~d^#VPz&YN137-Nd*CRFwS~SN9N#MS7tXh5L_l#y=X|lA?ensF5ZKk-o&L*(3W@uY zfxe63H{sOTQO}WNkM9jL9n!3@pGg2SdqRwJ?III=G=qO-lh3(ElJV%DUPv@$; zP3GftW8PEpXk5w>Cv@21j6;^78{6fk3rc1S9m=ex-P!_?uChMUqEDW?wV0L2xWjJh z%EAS9=`{-bVBjMOKixMZmv54osifkfJY$S~0Q>C`N$6%u+g?c(aGe5@5@fSW{SAte z?>h4G+-$^sx*={FI}en^rTEm_x-b$j@+l6v22{OzjsIZS={@pHDplWrN0Pg!jm5zT zeRIK6tL?yHB)*L#+ih71Nw8r6Le;9*MK_RYF?v7R1H{uKU=kkBN2NII=M+;@^{1Ti zLaix2N3E}(Nr5S;Y$Y@Q^ydHkaz4~wsW%eKnmMX6pIwB94nJY9hdlYzvIlsT43(JS zj%<`Y|1YD1$o#eV5CG|QUG3;~{v%;(8OBh{a^V=>w?cKY!{!-w z7S+Iff_?&M=~iLUmcfMuETE7S4lXDl0W{zop0~h*pOYq+m6@-S{dA(HK%a2!nEPjZOJn6BApz(yxjZY82Jof1s`o&{=0 zak&bJWZzA~>QP6@4IF85AH5`ypEP=-IVR3Seg#AzUTq^Q9=}m*!0Cp&{dE*xXcY_C z^3*MeiCC}c-V1sly<#RKC%>Ens_UJ4)F>{6<6jP5q_Ac7({L;D-W3R#GRtX~dMt>6 z9yj}EklO$V^lX+Il|ZaUJy@Ka)*|WOOhSW^AmX$?wj$s7e%fmFr=npK$h5lorBc@{ zr{udp%hoO3KpCY~3Dg)_cb#jXFo0f)B?RpO%!zTeUQBB%)`m}O4k3?co#UP)@6#@t za4^tHa5<|a2ZUaP66q&+sTA+g=7o*OxK{R#y6J7+6TUhh9bM@OxD53r6Yp|#6r)tXDUnggeTUJBC{p)smHms87`D&;Ul7u_ZyX-er7;#2$RNetn0rmur5!!Vf^X2Y@B>U2>+AsI~6 z>N6R`t~-opN3#v0@7*nkqALJQ1dzkzC2hB?*zWB9-O+X$zrzl#17KhPmXM9w{~pX@ z6AjwCzntp0vp_nu;ba#LX(OPZ8iZRb`d@PnlXNOgD?*53v!*d^DsNoZhFJW_@~W5d z$|&Zr)B55b#=qBrfSrk%d{Wv}P^J4uj=Y+M!w>q6J|5k7@Gz->L)-2Qlj^66v)#XF zZTlG~ZKn+&aDW=;`grimt#{mW)C?8Hx0x;IZ!v_*1*5NhgNX~#n3x8$+vro{~=Cs{0mq(Sy>eM`!lEXXBXcv+qHz%n@;SFG{cF0MqLL z!c-s!j&8+8k-WzHv7bGujEm=q7?LYFQH!E##b~=>!newQI@esG0n-5k02iayK(fB; z639VD_pq87N@!HYb>t0wBE77Nht;ox&Z=^*rLRwIC>olZpi^G4 zxAuNvl7ysa9Al(-O=H$q4p=AYt@FId!bUZm?v*@%oXS{9OunIGY#i~j;=KrQ**{WBhL}D3LjUPN$Wi5!OJbI~jE|(7a!$i#igf8JNJ9%1@iCO^=JXlj z$k#$#OP^NKT92~HMsOoMwucB|2Krl6Knw1nBu|OffaYgiwl}}x&WbA&lpT;&%UdwA<+J%_!IzsV45lY`b+$fNkG&V#X_d zLdV(pV`BbOZAQ{bfn8guCrjjPWH#l>{6jq(llB>*!i%&)#W}Msz4j+6u|Z|C#H?rqnt2q>dM-M?EPR+Br_fMa|;;#H_V_Msc6Z+BnN|> zuCb|dD>#}l1>M03OppvDdb|=h03;AC&tW}scaTB zvTT=8892exhTO(vLr<4q?=uhZa^M6Y-wDOY z-qxu5Og+pJmCqaOWe4YU`(i+uPJ*m+fAdY?Jt5bu+uN4E5)g@cU_E^yr5IG=&ul2w zBrhum`rdj}k_*K?r|?x4Y8M@&Ljon;BD@PWkH)1Il@)xj9PEftO9&Vicic{yf5KQ( zdf$a)9*uvaA@|!BFXMWXv@I`2kQtdVsZX(9`Dbr6IHjEUGweJX>_TetyAO?rsiEv2 z%QvdeM^PdLs-1hK&I#L6E5Xlu1@`?vZH@yzU5(y7wj3%h`%l$3OLgLId~!cx*HE3MMqOM{d&C?YIK2nZ}KEG7BY z|M~yUdGDO}zVn?scV_O~Ip_Pm_s*RULZeiRIbK^+nV68C5C8xWtEwpK0st5gG_Qw` ziKbVwX=CZk^cawIDP;_-^l0EKG4wDcdgec^vZ{tQldu3#NI*m!01y@u21^JDO8|wL zgn<$w;u0Vs00sc#0~VU16{G9Fhhp(XJ-Qc_iGlG9lE}jUIDpi9RNJB@F+qHuhw`I* z1(m74P{$cjE{Kg=AO-5$-=8{M6_war_qYzqbcL}YAkN*X691r1$Zsnpel*e*lG+s* z@wG;HYEKoA>yekEuRFPAG`wnr|CTA5_KSlfoo_}~?D18nazuTgci|S~XogIOh#Cu% z#T!ZoDbwhdlWKN;l7AUrT*FsAv@=A(VUc!~zMnASnXfKDBqieFD1sW&v2C0iR-%HMB5(!jBy*ea%#Q#HTCr~9b(AICl!>2j=)&|EXuEN)7rIxM zZp6a?O>+jj2U(~XCj!;LvN2~YJ-;a**Ut%$I#D8GY)IG7(~4+ir;2Z@0B9fd?eVm3fwpo6VQrYuf5@pF(c7$__R zd@dvc6wgHAco6}G_5bs$Gd*xw|9ihVAD0Xxa~#)#)zJ~HMbw^OR7eP|9wcnTZws;o z@!JD|w$JT=qCi`aeWn_o+W%Nwgy(@TFP7HLg|--6TXkhcz{9`tp{G0(Jwo86V&V$` z5K;WA7=YY-TJ#{EpQ?ru-YzCF4hL7vJE$!HKy96=Mfwb)luJ0u65v233IFk~^)jJu zrIio*i8leCg-Nu8Tgl=b3-kjfkJ>Rm>A(&1O_PlyxjX^(rIf7fXn|y) zRpu5roCUp0&ttMqbfe7(0K5o8GyjjA|1<7?WDXC_{3HM7|G&^bhx(7yFIjv!4b`p* z$sOwMI0j1cIXsf68~lhvEVRrXnpBXW8BTj7+Rw0+%_qOdpkN;e{m+$SlyIZ|2?bd` z(D~2}@5@-lh66sMFyEw-JDv}3j0EynHJO@xeS%g1yKMgYJ0jGbN4;t)T(1Z=0^z!! zH~;q8MV?eKpR0;Z-k_K(DP3D6bJmc$lqgtPtSQ$si}Na5aiNhO$5!jayRS2o0^3X1 z*to@WK~Dc;R#wSnW7OB@ve3gp7YFGYO=3Q+SG6-E9_}tShOKi>qSVXvZ=#sq_D1iy z<;D~wF{Cs3^|BE;C)b3ARUyNF*szlLUH{X$`lwFGEv|C^&$VM?>Gcafl@B;B8uHQvQmzG-hq?_nw8<=3U zeeSH7SwWjatcK5Iv^vee=7Z+Is%{lpWNRD+v2?pM?tX8um?N z03aFRBvC)QubVv?irdTSC+)Io?hp3M;Z1?&Liqffx&{W%D5&x>G7Qc2&2@E`=H?WO zb(}%D9hDcR@Hf!38?3BPJ$X#D)Va`@yq`O-(-7a)f;}Hf^F{uy^by z{*kZjq)Tt_F4F&Hn*Qg(&$Fbr4Wr9i!R*7OiY$>Y z%jsxSyx-fFm(`8$SU3us*%l7Tgydc^-zaEkB$t$wRCQJ**PdhVpV$iv#;kV^ycA4r zKfa+}ZZ#(HT&^gC{LC`P2|KkQ;_v(Re0IY0v-^+*>1Ag{#nst~EHvs!i1v|(KPBaF z8i4?-_Mpi2_8>`Tl?kuHB;N(iJMYQGVKOsI6BDrET20I%lxMBQe>1^AT_=X*SXRdW z)@{3sukHwt{u($PdU$PMyp=0CD9xR|Rt`EnSs`PM+w{1G+zeM2uX0+l_5FqmTBP*a z{+NwnAZovRGgBx7nY>|>*d`6|P3lK>F|XrY1jDkkHUh3Nj(bD{<_~1ooE$>j5gCQo z)MlBAEFDumE3JV~_U5hQ-%$2A?cCQ}5jNzt#5q5cYO05q8CE!%vMp@m%fBJT?yyMD z*Un>a7jbfe@Yj!`B6avRKNSrvY30-ZS*8LOS0_>?!NF_a(9vJYj5(5zHdtkrU@+K-I_a+S22mZ{aVOEat+O_{87@ zUOdP*9q}rA@qkHd2lnZr@4~4D>w(87!G^W9wJ<@8m8*%e`T6<6y`1fB#~BCLyU^Re zZX*2bVetJ>4_&CkaUE8_Twu_Gv$OMJw|B_ZN{HnXI0y{(w=V3{>^C)bH}AP!7_)N< zvLcG=Z{KEQ>fS>&NPzn)6}=93fC-F7<&bmhC{ojHIR}Ry^{$XXQ2>~&bC2Bk>6AopygCc4v%IhV^oiU+rF7h5sq^GwB`-boxbX71ox2EGm!0NC+}E@Q zxGWeHx0ggemWn#RK3*`EOjP91@b~)_id7_g!wClysW%hBre#9Oy-^pfhh1%7ks!B% z%#mrK2f;G5A^D&C`JMZ<#I4)6Kr%+zE(|YZ^@rGi%!_Ns^d9B$2?;#Jh8fw}UsiJ8 zr%m>a&2RhbI_m?0z;K?-F?*imasJOk)YP?lIG~V{|Yj?1@_F0Z&Ne8bT-jO61U~l9pElT z1uCaJ`&=ufuOJQKI{PBds}4_RVP=+D3!8=(?VR}jMaKZ^Mq|W6QvCxCdI=>VjvT3Q z4*CoLvPqE!0_3>din6Y|SD$~~zcv@kb#}Uh)@&W2?m}$B!lI%TCB^JpB3I>(dn$5S zJ{{`nWa0AC(bL`DZBQ-U29t}eg=v}@W8b)xYI9vYhQKXI)v)E}Ypk}ANun-iBS`;d z3NMRtg85C(MblzQIYs-ey{*ED`A_4Un}xf3sT*OGoIBj~xe7Wlj?`tdv$J3@xZ*T9 zZ=QrULQ}b^wa9Pyl1Cyt&byR?BBN2ct;kJs9zklr#n0c<@4AxvZa+pR%qK*kCrzwi z%8v330;QGZ;ZexaDK_$N^=CMmxHwv?lr~TNu2>`jF^sEy-X(7PgELkk>=WZ=yT!aH z1wu72-zlax;N+HFqCm#!{?#-fxiY7*GRJ%7=M}9Jp&TiVk<;?{o9j-*K+D|O<5YQ; z^by&!l}^AnJ1;_W zg-A`0I1*v}gjc+zH-2z9ysWIGw5-HJDllG!&Bevl*d*CE=SWJ|2|INj>EnlvMpuXc z)-7UU;&_6w`Am+vAM?^$Zz7kJ)J!4cOOBXT+UDk=hk}K17mULUpPHI@gf}okVxzNK zveMJn?Uh5~xC5kC5d~ZOFwA8Tdz*9fU(b@JZe0}*Z*Omcy2vO=+tECk6p2guQ97hX zg}zDA#11CMak(8lb!%`L z2x#}x%kM~jMG-$_$Jrz1iGq+*QEiPFot}3Hf18!zXYaXrHt_lY1FOjF;Y_C8Ie~@6 z;^Y&0GfE}l3r@{@tY*E*>Fe1-7kvr)RfsD>*e?0hlm2R z%U>So0A46QMrdIC=iCgP2f2MAwMg;saQ<>WO&&iX4IAQ+GW~-xhlD?^K5aV1@s(l3?CBoIz&~FTQuo$fE(YYkW3PLTVmc+O`t!R{1f6tL3{<8(w)era*_2a@ zx?ostbzj&lL@UsBdKCdF^jDG$^^UOGqbG%9^e!;?#ArhA+!uCfGri zkah4UwToaN4XjUpnX z*uk*$cnMx!-zQc@yolUEIOnr0AvI zTm@ObNF!Z+J=iGD!{{YVdU+cBTlM1Z>E$AbsGo75+sT%WG;KS+tPJ-8FMfRf@5u3M z8(UZ{WCQNP17*tIoT>!5#O+0y)qSjt%$*qySoTs*-`o2Wn+fEM?b@)ivtxD1X%Tgt z@*dmy*NuHI7ZmZ84BRAB7QAF{!hdr2R@JF}OT#%pDI zp&V8m?}v)LooGh(7d%c8<92HX`-Zj<&X=p5bWg*@W+#^wl?-Bj%%1M*S?bCQ&ok*7 z(j`#|s&bDpS1FtVn;CnPLYTmH3~t=I+KjQC!ohDJeVpaAKMz0R7GA?TVk&Rs z^IKWz>FLq8CiLg-+Gb4q10v&<#>TYSDRgho?BiI}MC$KoJI^KdI{hv38IEsh$tsDC z)?V3XpP+Geb`6lsVt#sChLnQnns+$;oFg+jq5r)sxkrHEENZ%x8-7nCnHN|u{c0aN zL+9pKwvwsFCgGjV+4HUQS<3klqIqigM`g2Zr$ZO4_Kt12mO-8Rv``y5{Ml==H4j!f zH|>Bx2jdO~GK^&y`B90n;VVAf&y3xyC8yl7<%}0zmuOz~+k%f1X zSGRQ4BnDl6Teq`Ri5&*CH8p29@q^ucoiTR(baaX@V{BmO{SP|YTu;Pm`zXBCE#)-v zJ&p-d#ALoe*R!xQ6b3FiL{RN6}DL#GRe#f_4zFo;$H+xt1GQRq%GWg;imc`68 z_0mA&e7+z;_HNolaI^F}uTXEhA1?3CrgpRnbdk|-4`4V0X?)#igY+@C*jHE}9dnIn3$K8Hgp(;Ti z600fNW{7#XMg>hb?zy&Ct6F(S<)w5KLfYpCImxOQ>V{HO#>Xbyms@1m_|nwg4O>_d zkQLSw!27Mi50J$zVj#l3Q=Tj5#JSU3Ejf8ykp&-AzMRRG#KO2{mUo%UQE|a>bqmrj zpf)aiZ(1}HLdS$D{}rZPpd`|k`Heyp689`Ec9=XQI&mjddscxX^N{1G{7l64nUEsp zH=EC|<1mULM7iUcF3na8J80yr4{S8oNQ)0uA?H?O(FZ3$BcnTWvH~tSb@BhmlcLZ5 zkI6|4e3UEC}2Mg!QP=xacO|{$m*S882hpoU%+}Kij_`!KiC^pb$uT8h$wk9 z6lS1fnSRP~D81_B9x71O*4P}oL+Hl%!45$;+YEV4&6&vZK}>>ePY0Ab6I4}MZA-&| zbxkIuQAzTZT<=!FpWsYE?63PoJ@g72=N}H*?UlDHk+ajS_YX^bLyH;tr+EabpnNH3 zbQZ{TgoQ5%l1TO?(<0BYcX4bQ@QvRJB0}aft}9{MS<2$d{(esbWkTN>tYjeWE-o1m z7>u!^;swXx4Y?a507->LJAxs{l*kb{N!z>EXwVj$P_Y2r{9pL(e}ixT7oq^gaC=zjt7NmoJu delta 6448 zcmY*d1yodByB<(r2&Ee-=^SF{5G19$22i>~x@H7Hz+nhSx}=e2=#~}{P^5&RC8R-W zr0#s*TK|9VJ@K5q_u20`d#(NEQ=&Jg!xpcvtNwt9fd~KqJkZooF$4f`-ecQFKwKRI^jSrPL_k7fQUHJ`NE9ps5|sf-vWiH^ zfW%}#BG}nDAMX7t^N-v=$F7LCDfTPTNF1C;uo8A^0^+w)D&@L?2IPxw)Nl3KriiL# zG9Z81#9cKEAxwC)QLkCxCIqz3){h`~N=NqwtbGjqQz-7zsVoPAeb7lk7tBG2fC3w* ztZ8xr@kNOrBheXS)o}%8Kwtrj7+qSXe9-aUwu>N4+mvd-8?3x4UbzXVq zjo}Y?{ug;_sRd`o>00u;xWoN!HPxRUvMH?o?KMSTKTb* zkv@6T{%jFI5c|m`1vMMKv^c%A(BqtSa|8JH_2pNB4vS0dJ_oxRoBNyVXv^4pH)P?y z!sQTsVJ{&K28#&VNjOOgii0HW1f@l#UI>ayh>6;RKz0sxAP`akuk3-8w4|uCln6*t zN?ckLd4lId43aed&sHNn@gM)+enU1sIZi_o@Hvo%0ZsoMyYu*VTIwqQ>Hq-qv;&63 zBWEQ60`B23_g;bk0Ga@#2>BzJN|vDmc?at4qlKL>^I@v92bMId->Fsc$jKP3d3P$P z$nR6@8-#jCa&)qwgo(wFWY1XJw=fu`19Y31pYut+-?;q6!-A$JYOKCpF0m@pVt=@X z_kSVopP~N(s|~jIzX1EE`6?JI{4X56`A}NCxS#unXjzGrh;Zk1Y;VN@|7exwUI2n^bafm0D9=~jiJ*R=G|~m zg-xB+&u@$XRiv><>MstrZ{R&oN+B9HnZiO1T?Vj0lCBli~cA zLVvcyZLYx4v$q5A5|Oh#u1IsgaCB|nuPB@@4FiMYyhm3CF@}cky4BC0Ze0V@XggAG z>bL~U*?FrA03A)4gH=vo2Qc+gEfm2T*B!m1ofXrSz?{PBp9o@fMO1|b6kbAM^bPpT zQUImo_WAbDaRlYce25)l>p%D6%wAfL7Zi425)2h~%fM8{V5*!l6;J&fJH8T5Iogvd zg$ox072O|_!Bc=00$O4GjtzYSbB&D#uf6OE9t)cl$yB$sqU~pFZ%+?neu+Fwq+Ql3 zAB&150YJl#D;#EOulzFeGe)apucBiwGyw*-O)M^i|xrWrko#-V+0EDbr|QFQQI<42G*|dV@12c&stM z3w!2Ix{q1d_=JHKmF{*$YlcQYwBBkx(;Cgn&UO{&L#P?i7kYTOIoQG6RM=B|ydrFR z7qpqY7r%t&6jODJFw-Kxef#!`&zP=R|kqaViQ6*FGUD-nJ46bocY~yW;8I2==3=r?=4ywo#f|&-j|~)qD9Z zKq}tNXg1ORr$PT5y!6;E{+iWLNakP3f%$=Aw9-2Kioz6xfIFwmy(}Bw$-cT08 z#eIC0tgg^4YX|$~gVup5!b0XQ=Kj2=MW2_9eowD9ms@scZeh9mjdV`BWL6)x>hnqv zp^K2@2w!GiMoo&MA1-foEX*vu`Lorr$CMwCEjqdM-2I&fM-u&kY@oN&B7=ZIl3k$Z z^3{ePTOjfrRDBbBJ~cfXpjCW^MBiebtw5c+}Cl&raBhf^u!R=+$_V{?~_CM9B1}!81{}= zc@ntKh3Z9)wK_X1D=}x?fBq0TwN9u?E49*?hQszKA>jDy*)lZf72aU=-84R zdcmCUJUu@j`nxantjEK}({sN%&(+oS*akJfEc|Lh;)ju#Q~Sh5sfu~v;T&!F?!!YF z0UG!9ztyWjhnT8`DKr`#?|sZi?Y`XPyD@1yoRqT`AxMJ1?)K!v(c9Vntj4HK!``O+ z2^dcZ1i1u%plod8V{Bq_*Zxd`=UR#sN(DD;JBrY9#)QT$U!$LU=&*SS|(;U2in zC@pnN$K_TD_px3_sB{6_s$}i*LG}30mW7Ms{%xUsL;WMyvA4Dus*XNkTg=}6^wRG9 zg#>90q}$ZQ1R;2Lbi3ZYC_gr`4?Au*fTf&vF-0L7)r`CRN6zFen0P-}DBSthK>k*u zP(&sgC11SeI&aSx8B>xD^Uh=)?PYN~?7haXz9rW+D}u#icM=#zlC~~)4$aK@lT^5* zrDdC8clq)++|6KOtp-0@UB!gNR^O|gncQp_e}?AqVgk-ZFY4$)!c|D zHQ_8r+e%b_4Y*Qpac=_7{H0*9L^^4-FRE*{lM|~VavOLv;dNP{@w+eiDr<_hFiJ6dN~5CuoU$VAcZG#(X0RSn z39vGmL;j(V?&=kqG;juK@ZcdGh;urF9NgS2nb}8Xq%4Tf?N3|44|r-+3($ zl#P_TIqT*QpB2^Xe?(=sc>4tA3UF}^IW-_DjlyRZBb3SmoMANKs6h?!%=#yhOXY_`sZlu8-!P;nts$x)wTxo|957m98&kpYp`_aiqtmt~$g1Sya#*?ef zC$$VGaxSITi^MNCu5z&vhJk^BlSAfGzv#i~`PG5iG!^-Na#XRn`N+4=D~DUc--pcn zuOSl_o@{J;a9FN#ft1vIz*A@P`xZ@sF%A6l#gi;Wv;DfC&aSFFhx8g7l!Bt-KA)s& zb){FSrE(7_PEWH8UhxRkAhGISNHiv(T~bow?(S}|v5Pfb5Fq!B(-!FXWRSnSk58M4 zl~)8W%s*B$&>r!?AZBuMayNXY%Cg&(ZjitB0%aHi`!RiLrd0d<-b$gU3O(4%+}7-H zqs`%BX@9=IY5t0~VRL-D5o{*uTopXnOj(;TETvF{s0MSJHQZRdsLWLii_|Tf$dc`f z@cA%}9g1nxN2xa0EV>n_WPt1?s9n7gRQBnP> zk7)uo!RXW}wj4^Ye?^6vqU}O%gl$v>O(ITlx2~Qf^DigONox-{K^3*22QIsXWwq{Z zUjJkbQZs<7@nJ_4vcG0O%)lVy*TDPIQm#a2s@Lh5m`5MQ8$hY=-&NJ_f))}69rk~= z-Ax@UeiayB^54`eeX>Kkp zDJ?50%~$k`)ZlP)b2qm@_+{_N7&_lin%REok99}Utq*x0A`l48cwPz0>fPO)d`_66 z`aV0SprgGzf{%)p*7gQ|5!n@6A6s5gk(dztDRbsA_N9Y^)X2|XUz9v)SV3ht1`?N@ zw<+Stb%!uEF(w=l6yoI_1lRfuyM_Fiz`0V0o4L14Zd_5dT+%@Cj-pfBUP6{zcJ{Ed z2icXq%W*-4Eq&zFJzhZk<+w?vV3Xf@o{$anet$R{Zr(UG>RG{mh^(!wh9lzdJQtlF zN5H#eNvEbx6E_kAezx0lic1iU2o8<2ZEQm&vv!=1-*LPpoQ~AyXsU6)MmZmmNTWzJ zG<%*)1!}tt2%-f91?YcgHpp#yE_V{kh7R^FeW&49WAY}Ktk1K0hLfTp2YZPQTBl(# zFff2Jb-|(O+~zXHWlpD^Pw8~JR&srI^ScQoJ4{rHWrdzA!qW@mJeHYU$d(D%+3!1B z^!rpls3g!m5c9;FFDj{~4^+II8|lZogv*aGFsskirgdv}aY+fccB3*i4HT1gCc&7^ ztCn7rZI$I40g4+aS%R;@C|Jg>LL?MzUZ{o)c3RlXK^kh5+3)miM|!~?{2d$9=YBoq zMpUS5BCMfXR|u<7S+F|q3%CP5QrzICX!vLE>_|+5CAT89jUk41CvazB#?#s$cR$C+r&fr#YE4~thp9zS9AB0^kj$C`#~KOE3G zgb%@BFe7x4;`m5cuU#2mk4MvC@-&nATCy*xcTQa)xtOEI{p)j(Pk>JFpnF0z!xk|^ zlwxwIbq=>&M3cZ%?PdvO2ce=TDrC0ZXc9|@{$@1 zB{HKjs!B#_D=I!hgCzZv-?5*+#O6|eO{b=B)*M64$itK&_6|OP}Ie8F29ccGC z%FhknG>9LW1x&;SCOy3Tg**5B2YLniLsnK^3vJR0(qCYHy^y_wInCKSy6<*&=ZVd^ z%n@~{cWg^85k^LoJ5>tQ;Wd@jS7s1n{E1A-UAs?hVM)r=1iWiGbE8vv{_yDN*3vZm z=Ig+3xrfI>@VGb<1C!pgG7kAeRLD|~S)stQStK3%cG0$0`MHnXW8C`i-Ug?^TGP$u z{kxw!K7Fw6LRD4OmVI;rLdN91nUDL=fM;7`SJ=4qqKuLacp@Q29AJu_O?x9`@BWZQ z#6#NYSoyvZ-_c}No%ocbkJasq!bjtfQ>^MLcHVt1>g^{;!pB>p=wtMhqS@jk4;BPo zoP-q5p5Ikah`G{4DMp7J-nU~-6V?z1gX?;nVa91Qin^5WcLP)4<8Tj4m|3qtQ{V`_bG?&;BTRhj!wA=^+JZo=)LUQ`MJ4m>9Eyok{szI z#}Y?(vuZ`Zo$E9mSs&RtSnu@O=~RMP*?19=eW9IRz7&7wnxk-0+039zs9COpEPa=a z2P~O#w5O;E$~Ub7?D4(2=_6z-m}`X+yZ!!^mX@}-aA8ktyecBD$%W*Wl|Gu(5q#i< zskw*$`ML9T=h)ZzUk%hvp?c#T5C~kGz|Rhay602f$!W9XtanHMQda!a-A-lt-B6Ut zw6U-AM7ZGf&ZgX0*vL+{WE2vaH#jsv0K;3dYznV#Dv5B>kizEKT<(J?aA!JnKa5Me zu}??yX7M5en;Z4E9LUAPLa&@YpA1&g2p?_M|9Wwk)JH)$fOuUhFnU3$f3-_(eVoLa&^J{GRmHuC;jp z)DC|aZ6bslSr;)&gC;ttaM@)Y$mGI67K?aaR?lMeE|~elC^p znCZH7TgDL57PK%$Zgx!nYhH4Vxc=HkNz3Yo;!3y~H{6Bvhs;(G5o6}g(Fe+SA;^@D zccWL?>(hlmPhY>F-xW08^E>uiAo8Hf*+NXGv(gI@5R>285GP-gd&nZzE7#!R`Fr0Y zR>-KS<%3omtM*>$y6?bte*0vf^9Zb|vc1h=M(HEim#rRD82r+Ea7@8@nqo_WT|yXT z2EDo*aMcfigBuC*9GY4Ps%^2oe9tHZ34aPTrRPLys_QW?H3c!|^LUC5tZzaiQ z3FL1PHZz}Y3&%B#TN^V*758MdB(^@A$Y$WIS*zXD;nX^B5+kj@UV37pbGDiLN%3aF zT!(I05DYQV3Ak0BATU^sO+480BP2lU6_UrREZsVV6`VpqlIJ7+COyIHPJF(nb~&jV zkx#>z2bcP--C;fQ$Q&lG82qfv_50cy9PbiHZdG~t9(MX*JT!unvpa}eYA5-4J`0=w z!tk6gy`3KYtSr}|5!O1>za5uanQlKEwef3X#@n2cf<~p}%UVs@8{mYx^TP8wE2uD0 zk9FqcJ`FZGRd`zy;pFH0ILV+-HjR`ecO!OaXs9LgCk7UR0Z%vWnM&lU6-}1!Je_VG zA@D|9J9R>**&L_7qB)LcQoY2_FhA1KIq0>*cUf7Mn!K7JA<{CrrA4x+9}`t}Se`;N zEZ4@+5bKPv_1c$_5&DEjV^@AZnKHCDvGqrr^kHSC`;OagadFAEe^y#^>6=Y`^mpxo zm=OmjSaHut$VGQ)dzM<+$8sJ<273Q?sjP4Dw6V@-Z3I1w`#UFPJ5(`<*XRUJaLiWC zkvS{LKxM5hH_IRH#2^(%WoEg~prJk2=ljqk;73hY_K(z6^#=wzq57(ZmQQgG9eaR4 zC0peH{4unRn%-OvSXY20&50U+JpcG9FKwnatU2F-K>ps^^e7pj$}mpRBZ34Wj~b1d z(U9rn^a&o!yrIQ}6DyEivzRAdf0~B~pJE3~4a(==JpM$i1RO`bX%-Ew*wjOXk#aG>)Q3k^f= z40P}XMW%zB6Y6VN%0@8pb`Hy`{C!vvLJ=P{AE$+77`F6mvGm-=#s&c&U-HONGyW(s zpql`jL509f7%vkAg}p|d?g0QmF|s-W0APR^%eVd)$b_-I{{nUuScdlh!+-hPf0^9> f2+{*STwRm)@`#rcP9{@hr2v|$x+*ow&!heaB6>OQ From d6cf48544a712da040a43b2cd0553bb82334d34f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 12 Feb 2025 22:49:16 -0800 Subject: [PATCH 020/226] Get existing tests passing. --- borgmatic/hooks/credential/parse.py | 2 +- borgmatic/hooks/credential/systemd.py | 28 +++++++++++++-------- tests/unit/hooks/credential/test_parse.py | 22 ++++++++++++---- tests/unit/hooks/credential/test_systemd.py | 8 +++--- 4 files changed, 39 insertions(+), 21 deletions(-) diff --git a/borgmatic/hooks/credential/parse.py b/borgmatic/hooks/credential/parse.py index 9125d235..699f90b7 100644 --- a/borgmatic/hooks/credential/parse.py +++ b/borgmatic/hooks/credential/parse.py @@ -42,5 +42,5 @@ def resolve_credential(value): raise ValueError(f'Cannot load credential with invalid syntax "{value}"') return borgmatic.hooks.dispatch.call_hook( - 'load_credential', {}, hook_name, credential_parameters + 'load_credential', {}, hook_name, tuple(credential_parameters) ) diff --git a/borgmatic/hooks/credential/systemd.py b/borgmatic/hooks/credential/systemd.py index 6e3630c7..8289c6dc 100644 --- a/borgmatic/hooks/credential/systemd.py +++ b/borgmatic/hooks/credential/systemd.py @@ -5,29 +5,35 @@ import re logger = logging.getLogger(__name__) -SECRET_NAME_PATTERN = re.compile(r'^\w+$') -SECRETS_DIRECTORY = '/run/secrets' +CREDENTIAL_NAME_PATTERN = re.compile(r'^\w+$') def load_credential(hook_config, config, credential_parameters): ''' Given the hook configuration dict, the configuration dict, and a credential parameters tuple - containing a secret name to load, read the secret from the corresponding container secrets file - and return it. + containing a credential name to load, read the credential from the corresponding systemd + credential file and return it. - Raise ValueError if the credential parameters is not one element, the secret name is invalid, or - the secret file cannot be read. + Raise ValueError if the systemd CREDENTIALS_DIRECTORY environment variable is not set, the + credential name is invalid, or the credential file cannot be read. ''' try: - (secert_name,) = credential_parameters + (credential_name,) = credential_parameters except ValueError: - raise ValueError(f'Cannot load invalid secret name: "{' '.join(credential_parameters)}"') + raise ValueError(f'Cannot load invalid credential name: "{' '.join(credential_parameters)}"') - if not SECRET_NAME_PATTERN.match(SECRET_NAME): - raise ValueError(f'Cannot load invalid secret name: "{credential_name}"') + credentials_directory = os.environ.get('CREDENTIALS_DIRECTORY') + + if not credentials_directory: + raise ValueError( + f'Cannot load credential "{credential_name}" because the systemd CREDENTIALS_DIRECTORY environment variable is not set' + ) + + if not CREDENTIAL_NAME_PATTERN.match(credential_name): + raise ValueError(f'Cannot load invalid credential name "{credential_name}"') try: - with open(os.path.join(SECRETS_DIRECTORY, credential_name)) as credential_file: + with open(os.path.join(credentials_directory, credential_name)) as credential_file: return credential_file.read().rstrip(os.linesep) except (FileNotFoundError, OSError) as error: logger.warning(error) diff --git a/tests/unit/hooks/credential/test_parse.py b/tests/unit/hooks/credential/test_parse.py index 9c2ca8e3..5fd4301a 100644 --- a/tests/unit/hooks/credential/test_parse.py +++ b/tests/unit/hooks/credential/test_parse.py @@ -4,7 +4,7 @@ from flexmock import flexmock from borgmatic.hooks.credential import parse as module -def test_resolve_credential_passes_through_string_without_credential_tag(): +def test_resolve_credential_passes_through_string_without_credential(): module.resolve_credential.cache_clear() flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').never() @@ -19,7 +19,7 @@ def test_resolve_credential_passes_through_none(): @pytest.mark.parametrize('invalid_value', ('{credential}', '{credential }', '{credential systemd}')) -def test_resolve_credential_with_invalid_credential_tag_raises(invalid_value): +def test_resolve_credential_with_invalid_credential_raises(invalid_value): module.resolve_credential.cache_clear() flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').never() @@ -27,25 +27,37 @@ def test_resolve_credential_with_invalid_credential_tag_raises(invalid_value): module.resolve_credential(invalid_value) -def test_resolve_credential_with_valid_credential_tag_loads_credential(): +def test_resolve_credential_with_valid_credential_loads_credential(): module.resolve_credential.cache_clear() flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( 'load_credential', {}, 'systemd', - 'mycredential', + ('mycredential',), ).and_return('result').once() assert module.resolve_credential('{credential systemd mycredential}') == 'result' +def test_resolve_credential_with_valid_credential_and_quoted_parameters_loads_credential(): + module.resolve_credential.cache_clear() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( + 'load_credential', + {}, + 'systemd', + ('my credential',), + ).and_return('result').once() + + assert module.resolve_credential('{credential systemd "my credential"}') == 'result' + + def test_resolve_credential_caches_credential_after_first_call(): module.resolve_credential.cache_clear() flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( 'load_credential', {}, 'systemd', - 'mycredential', + ('mycredential',), ).and_return('result').once() assert module.resolve_credential('{credential systemd mycredential}') == 'result' diff --git a/tests/unit/hooks/credential/test_systemd.py b/tests/unit/hooks/credential/test_systemd.py index 97b2877f..d2f70d40 100644 --- a/tests/unit/hooks/credential/test_systemd.py +++ b/tests/unit/hooks/credential/test_systemd.py @@ -13,7 +13,7 @@ def test_load_credential_without_credentials_directory_raises(): ) with pytest.raises(ValueError): - module.load_credential(hook_config={}, config={}, credential_name='mycredential') + module.load_credential(hook_config={}, config={}, credential_parameters=('mycredential',)) def test_load_credential_with_invalid_credential_name_raises(): @@ -22,7 +22,7 @@ def test_load_credential_with_invalid_credential_name_raises(): ) with pytest.raises(ValueError): - module.load_credential(hook_config={}, config={}, credential_name='../../my!@#$credential') + module.load_credential(hook_config={}, config={}, credential_parameters=('../../my!@#$credential',)) def test_load_credential_reads_named_credential_from_file(): @@ -35,7 +35,7 @@ def test_load_credential_reads_named_credential_from_file(): builtins.should_receive('open').with_args('/var/mycredential').and_return(credential_stream) assert ( - module.load_credential(hook_config={}, config={}, credential_name='mycredential') + module.load_credential(hook_config={}, config={}, credential_parameters=('mycredential',)) == 'password' ) @@ -48,4 +48,4 @@ def test_load_credential_with_file_not_found_error_raises(): builtins.should_receive('open').with_args('/var/mycredential').and_raise(FileNotFoundError) with pytest.raises(ValueError): - module.load_credential(hook_config={}, config={}, credential_name='mycredential') + module.load_credential(hook_config={}, config={}, credential_parameters=('mycredential',)) From 92e87d839da779808537a1c18835ad8c1d7a75a0 Mon Sep 17 00:00:00 2001 From: Dmitrii Tishchenko Date: Thu, 13 Feb 2025 16:12:58 +0000 Subject: [PATCH 021/226] Fix path handling error when handling btrfs '/' submodule --- borgmatic/hooks/data_source/btrfs.py | 8 +-- tests/unit/hooks/data_source/test_btrfs.py | 57 ++++++++++++++++++++++ 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/borgmatic/hooks/data_source/btrfs.py b/borgmatic/hooks/data_source/btrfs.py index a7d8f2f5..d703d332 100644 --- a/borgmatic/hooks/data_source/btrfs.py +++ b/borgmatic/hooks/data_source/btrfs.py @@ -299,9 +299,11 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d logger.debug(error) return - # Strip off the subvolume path from the end of the snapshot path and then delete the - # resulting directory. - shutil.rmtree(snapshot_path.rsplit(subvolume.path, 1)[0]) + # Remove snapshot parent directory if it still exists (might not exist if snapshot was for '/') + snapshot_parent_dir = snapshot_path.rsplit(subvolume.path, 1)[0] + if os.path.isdir(snapshot_parent_dir): + shutil.rmtree(snapshot_parent_dir) + continue def make_data_source_dump_patterns( diff --git a/tests/unit/hooks/data_source/test_btrfs.py b/tests/unit/hooks/data_source/test_btrfs.py index c69d5ac1..8e7055c2 100644 --- a/tests/unit/hooks/data_source/test_btrfs.py +++ b/tests/unit/hooks/data_source/test_btrfs.py @@ -520,6 +520,18 @@ def test_remove_data_source_dumps_deletes_snapshots(): flexmock(module).should_receive('delete_snapshot').with_args( 'btrfs', '/mnt/subvol2/.borgmatic-5678/mnt/subvol2' ).never() + flexmock(module.os.path).should_receive('isdir').with_args( + '/mnt/subvol1/.borgmatic-1234' + ).and_return(True) + flexmock(module.os.path).should_receive('isdir').with_args( + '/mnt/subvol1/.borgmatic-5678' + ).and_return(True) + flexmock(module.os.path).should_receive('isdir').with_args( + '/mnt/subvol2/.borgmatic-1234' + ).and_return(True) + flexmock(module.os.path).should_receive('isdir').with_args( + '/mnt/subvol2/.borgmatic-5678' + ).and_return(True) flexmock(module.shutil).should_receive('rmtree').with_args( '/mnt/subvol1/.borgmatic-1234' ).once() @@ -846,3 +858,48 @@ def test_remove_data_source_dumps_with_delete_snapshot_called_process_error_bail borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) + + +def test_remove_data_source_dumps_with_root_subvolume(): + config = {'btrfs': {}} + flexmock(module).should_receive('get_subvolumes').and_return( + (module.Subvolume('/', contained_patterns=(Pattern('/etc'),)),) + ) + + flexmock(module).should_receive('make_snapshot_path').with_args('/').and_return( + '/.borgmatic-1234' + ) + + flexmock(module.borgmatic.config.paths).should_receive( + 'replace_temporary_subdirectory_with_glob' + ).with_args( + '/.borgmatic-1234', + temporary_directory_prefix=module.BORGMATIC_SNAPSHOT_PREFIX, + ).and_return( + '/.borgmatic-*' + ) + + flexmock(module.glob).should_receive('glob').with_args('/.borgmatic-*').and_return( + ('/.borgmatic-1234', '/.borgmatic-5678') + ) + + flexmock(module.os.path).should_receive('isdir').with_args('/.borgmatic-1234').and_return( + True + ).and_return(False) + flexmock(module.os.path).should_receive('isdir').with_args('/.borgmatic-5678').and_return( + True + ).and_return(False) + + flexmock(module).should_receive('delete_snapshot').with_args('btrfs', '/.borgmatic-1234').once() + flexmock(module).should_receive('delete_snapshot').with_args('btrfs', '/.borgmatic-5678').once() + + flexmock(module.os.path).should_receive('isdir').with_args('').and_return(False) + + flexmock(module.shutil).should_receive('rmtree').never() + + module.remove_data_source_dumps( + hook_config=config['btrfs'], + config=config, + borgmatic_runtime_directory='/run/borgmatic', + dry_run=False, + ) From 653d8c0946bbc299b8a259bc0ccd6b10c961022f Mon Sep 17 00:00:00 2001 From: Dmitrii Tishchenko Date: Thu, 13 Feb 2025 21:44:45 +0000 Subject: [PATCH 022/226] Remove unneeded 'continue' --- borgmatic/hooks/data_source/btrfs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/borgmatic/hooks/data_source/btrfs.py b/borgmatic/hooks/data_source/btrfs.py index d703d332..99e75b26 100644 --- a/borgmatic/hooks/data_source/btrfs.py +++ b/borgmatic/hooks/data_source/btrfs.py @@ -303,7 +303,6 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d snapshot_parent_dir = snapshot_path.rsplit(subvolume.path, 1)[0] if os.path.isdir(snapshot_parent_dir): shutil.rmtree(snapshot_parent_dir) - continue def make_data_source_dump_patterns( From 5dda9c8ee5da31887626ba3c3211f5a2cbf90169 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 13 Feb 2025 16:38:50 -0800 Subject: [PATCH 023/226] Add unit tests for new credential hooks. --- borgmatic/hooks/credential/file.py | 7 +-- borgmatic/hooks/credential/keepassxc.py | 9 ++-- borgmatic/hooks/credential/parse.py | 12 ++--- borgmatic/hooks/credential/systemd.py | 4 +- tests/unit/hooks/credential/test_container.py | 42 ++++++++++++++++ tests/unit/hooks/credential/test_file.py | 50 +++++++++++++++++++ tests/unit/hooks/credential/test_keepassxc.py | 38 ++++++++++++++ tests/unit/hooks/credential/test_systemd.py | 14 +++++- 8 files changed, 159 insertions(+), 17 deletions(-) create mode 100644 tests/unit/hooks/credential/test_container.py create mode 100644 tests/unit/hooks/credential/test_file.py create mode 100644 tests/unit/hooks/credential/test_keepassxc.py diff --git a/borgmatic/hooks/credential/file.py b/borgmatic/hooks/credential/file.py index e50ff199..f9481954 100644 --- a/borgmatic/hooks/credential/file.py +++ b/borgmatic/hooks/credential/file.py @@ -1,8 +1,5 @@ import logging import os -import shlex - -import borgmatic.execute logger = logging.getLogger(__name__) @@ -18,7 +15,7 @@ def load_credential(hook_config, config, credential_parameters): try: (credential_path,) = credential_parameters except ValueError: - raise ValueError(f'Cannot load credential with invalid credential path: "{' '.join(credential_parameters)}"') + raise ValueError(f'Cannot load invalid credential: "{' '.join(credential_parameters)}"') try: with open(credential_path) as credential_file: @@ -26,4 +23,4 @@ def load_credential(hook_config, config, credential_parameters): except (FileNotFoundError, OSError) as error: logger.warning(error) - raise ValueError(f'Cannot load credential "{credential_name}" from file: {error.filename}') + raise ValueError(f'Cannot load credential file: {error.filename}') diff --git a/borgmatic/hooks/credential/keepassxc.py b/borgmatic/hooks/credential/keepassxc.py index 5543a0ce..e12ef544 100644 --- a/borgmatic/hooks/credential/keepassxc.py +++ b/borgmatic/hooks/credential/keepassxc.py @@ -1,6 +1,5 @@ import logging import os -import shlex import borgmatic.execute @@ -18,10 +17,14 @@ def load_credential(hook_config, config, credential_parameters): try: (database_path, attribute_name) = credential_parameters except ValueError: - raise ValueError(f'Cannot load credential with invalid KeePassXC database path and attribute name: "{' '.join(credential_parameters)}"') + raise ValueError( + f'Cannot load credential with invalid KeePassXC database path and attribute name: "{' '.join(credential_parameters)}"' + ) if not os.path.exists(database_path): - raise ValueError(f'Cannot load credential because KeePassXC database path does not exist: {database_path}') + raise ValueError( + f'Cannot load credential because KeePassXC database path does not exist: {database_path}' + ) return borgmatic.execute.execute_command_and_capture_output( ( diff --git a/borgmatic/hooks/credential/parse.py b/borgmatic/hooks/credential/parse.py index 699f90b7..67d888db 100644 --- a/borgmatic/hooks/credential/parse.py +++ b/borgmatic/hooks/credential/parse.py @@ -7,9 +7,7 @@ import borgmatic.hooks.dispatch IS_A_HOOK = False -CREDENTIAL_PATTERN = re.compile( - r'\{credential( +(?P.*))?\}' -) +CREDENTIAL_PATTERN = re.compile(r'\{credential( +(?P.*))?\}') @functools.cache @@ -31,12 +29,12 @@ def resolve_credential(value): if not matcher: return value - contents = matcher.group('contents') - - if not contents: + hook_and_parameters = matcher.group('hook_and_parameters') + + if not hook_and_parameters: raise ValueError(f'Cannot load credential with invalid syntax "{value}"') - (hook_name, *credential_parameters) = shlex.split(contents) + (hook_name, *credential_parameters) = shlex.split(hook_and_parameters) if not credential_parameters: raise ValueError(f'Cannot load credential with invalid syntax "{value}"') diff --git a/borgmatic/hooks/credential/systemd.py b/borgmatic/hooks/credential/systemd.py index 8289c6dc..085d9044 100644 --- a/borgmatic/hooks/credential/systemd.py +++ b/borgmatic/hooks/credential/systemd.py @@ -20,7 +20,9 @@ def load_credential(hook_config, config, credential_parameters): try: (credential_name,) = credential_parameters except ValueError: - raise ValueError(f'Cannot load invalid credential name: "{' '.join(credential_parameters)}"') + raise ValueError( + f'Cannot load invalid credential name: "{' '.join(credential_parameters)}"' + ) credentials_directory = os.environ.get('CREDENTIALS_DIRECTORY') diff --git a/tests/unit/hooks/credential/test_container.py b/tests/unit/hooks/credential/test_container.py new file mode 100644 index 00000000..977f2f20 --- /dev/null +++ b/tests/unit/hooks/credential/test_container.py @@ -0,0 +1,42 @@ +import io +import sys + +import pytest +from flexmock import flexmock + +from borgmatic.hooks.credential import container as module + + +@pytest.mark.parametrize('credential_parameters', ((), ('foo', 'bar'))) +def test_load_credential_with_invalid_credential_parameters_raises(credential_parameters): + with pytest.raises(ValueError): + module.load_credential( + hook_config={}, config={}, credential_parameters=credential_parameters + ) + + +def test_load_credential_with_invalid_secret_name_raises(): + with pytest.raises(ValueError): + module.load_credential( + hook_config={}, config={}, credential_parameters=('this is invalid',) + ) + + +def test_load_credential_reads_named_secret_from_file(): + credential_stream = io.StringIO('password') + credential_stream.name = '/run/secrets/mysecret' + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('/run/secrets/mysecret').and_return(credential_stream) + + assert ( + module.load_credential(hook_config={}, config={}, credential_parameters=('mysecret',)) + == 'password' + ) + + +def test_load_credential_with_file_not_found_error_raises(): + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('/run/secrets/mysecret').and_raise(FileNotFoundError) + + with pytest.raises(ValueError): + module.load_credential(hook_config={}, config={}, credential_parameters=('mysecret',)) diff --git a/tests/unit/hooks/credential/test_file.py b/tests/unit/hooks/credential/test_file.py new file mode 100644 index 00000000..219fcec7 --- /dev/null +++ b/tests/unit/hooks/credential/test_file.py @@ -0,0 +1,50 @@ +import io +import sys + +import pytest +from flexmock import flexmock + +from borgmatic.hooks.credential import file as module + + +@pytest.mark.parametrize('credential_parameters', ((), ('foo', 'bar'))) +def test_load_credential_with_invalid_credential_parameters_raises(credential_parameters): + with pytest.raises(ValueError): + module.load_credential( + hook_config={}, config={}, credential_parameters=credential_parameters + ) + + +def test_load_credential_with_invalid_credential_name_raises(): + with pytest.raises(ValueError): + module.load_credential( + hook_config={}, config={}, credential_parameters=('this is invalid',) + ) + + +def test_load_credential_reads_named_credential_from_file(): + credential_stream = io.StringIO('password') + credential_stream.name = '/credentials/mycredential' + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('/credentials/mycredential').and_return( + credential_stream + ) + + assert ( + module.load_credential( + hook_config={}, config={}, credential_parameters=('/credentials/mycredential',) + ) + == 'password' + ) + + +def test_load_credential_with_file_not_found_error_raises(): + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('/credentials/mycredential').and_raise( + FileNotFoundError + ) + + with pytest.raises(ValueError): + module.load_credential( + hook_config={}, config={}, credential_parameters=('/credentials/mycredential',) + ) diff --git a/tests/unit/hooks/credential/test_keepassxc.py b/tests/unit/hooks/credential/test_keepassxc.py new file mode 100644 index 00000000..f73e9de8 --- /dev/null +++ b/tests/unit/hooks/credential/test_keepassxc.py @@ -0,0 +1,38 @@ +import pytest +from flexmock import flexmock + +from borgmatic.hooks.credential import keepassxc as module + + +@pytest.mark.parametrize('credential_parameters', ((), ('foo',), ('foo', 'bar', 'baz'))) +def test_load_credential_with_invalid_credential_parameters_raises(credential_parameters): + flexmock(module.borgmatic.execute).should_receive('execute_command_and_capture_output').never() + + with pytest.raises(ValueError): + module.load_credential( + hook_config={}, config={}, credential_parameters=credential_parameters + ) + + +def test_load_credential_with_missing_database_raises(): + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.execute).should_receive('execute_command_and_capture_output').never() + + with pytest.raises(ValueError): + module.load_credential( + hook_config={}, config={}, credential_parameters=('database.kdbx', 'mypassword') + ) + + +def test_load_credential_with_present_database_fetches_password_from_keepassxc(): + flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return('password').once() + + assert ( + module.load_credential( + hook_config={}, config={}, credential_parameters=('database.kdbx', 'mypassword') + ) + == 'password' + ) diff --git a/tests/unit/hooks/credential/test_systemd.py b/tests/unit/hooks/credential/test_systemd.py index d2f70d40..603c8569 100644 --- a/tests/unit/hooks/credential/test_systemd.py +++ b/tests/unit/hooks/credential/test_systemd.py @@ -7,6 +7,16 @@ from flexmock import flexmock from borgmatic.hooks.credential import systemd as module +@pytest.mark.parametrize('credential_parameters', ((), ('foo', 'bar'))) +def test_load_credential_with_invalid_credential_parameters_raises(credential_parameters): + flexmock(module.os.environ).should_receive('get').never() + + with pytest.raises(ValueError): + module.load_credential( + hook_config={}, config={}, credential_parameters=credential_parameters + ) + + def test_load_credential_without_credentials_directory_raises(): flexmock(module.os.environ).should_receive('get').with_args('CREDENTIALS_DIRECTORY').and_return( None @@ -22,7 +32,9 @@ def test_load_credential_with_invalid_credential_name_raises(): ) with pytest.raises(ValueError): - module.load_credential(hook_config={}, config={}, credential_parameters=('../../my!@#$credential',)) + module.load_credential( + hook_config={}, config={}, credential_parameters=('../../my!@#$credential',) + ) def test_load_credential_reads_named_credential_from_file(): From b283e379d0f5014bec187d80cb1c8738ab9da3f6 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 14 Feb 2025 10:15:52 -0800 Subject: [PATCH 024/226] Actually pass the current configuration to credential hooks. --- borgmatic/borg/environment.py | 2 +- borgmatic/hooks/credential/parse.py | 94 +++++++++++++++++-- borgmatic/hooks/data_source/mariadb.py | 42 ++++++--- borgmatic/hooks/data_source/mongodb.py | 28 ++++-- borgmatic/hooks/data_source/mysql.py | 44 ++++++--- borgmatic/hooks/data_source/postgresql.py | 38 ++++---- borgmatic/hooks/monitoring/ntfy.py | 6 +- borgmatic/hooks/monitoring/pagerduty.py | 2 +- borgmatic/hooks/monitoring/pushover.py | 6 +- borgmatic/hooks/monitoring/zabbix.py | 12 ++- tests/unit/borg/test_environment.py | 9 +- tests/unit/hooks/credential/test_parse.py | 45 +++++++-- tests/unit/hooks/data_source/test_mariadb.py | 66 +++++++------ tests/unit/hooks/data_source/test_mongodb.py | 26 ++--- tests/unit/hooks/data_source/test_mysql.py | 62 +++++++----- .../unit/hooks/data_source/test_postgresql.py | 88 ++++++++--------- tests/unit/hooks/monitoring/test_ntfy.py | 28 +++--- tests/unit/hooks/monitoring/test_pagerduty.py | 12 +-- tests/unit/hooks/monitoring/test_pushover.py | 28 +++--- tests/unit/hooks/monitoring/test_zabbix.py | 36 +++---- 20 files changed, 436 insertions(+), 238 deletions(-) diff --git a/borgmatic/borg/environment.py b/borgmatic/borg/environment.py index 5411623d..6e77552e 100644 --- a/borgmatic/borg/environment.py +++ b/borgmatic/borg/environment.py @@ -41,7 +41,7 @@ def make_environment(config): value = config.get(option_name) if option_name in CREDENTIAL_OPTIONS and value is not None: - value = borgmatic.hooks.credential.parse.resolve_credential(value) + value = borgmatic.hooks.credential.parse.resolve_credential(value, config) if value is not None: environment[environment_variable_name] = str(value) diff --git a/borgmatic/hooks/credential/parse.py b/borgmatic/hooks/credential/parse.py index 67d888db..2b7b2543 100644 --- a/borgmatic/hooks/credential/parse.py +++ b/borgmatic/hooks/credential/parse.py @@ -7,17 +7,97 @@ import borgmatic.hooks.dispatch IS_A_HOOK = False +class Hash_adapter: + ''' + A Hash_adapter instance wraps an unhashable object and pretends it's hashable. This is intended + for passing to a @functools.cache-decorated function to prevent it from complaining that an + argument is unhashable. It should only be used for arguments that you don't want to actually + impact the cache hashing, because Hash_adapter doesn't actually hash the object's contents. + + Example usage: + + @functools.cache + def func(a, b): + print(a, b.actual_value) + return a + + func(5, Hash_adapter({1: 2, 3: 4})) # Calls func(), prints, and returns. + func(5, Hash_adapter({1: 2, 3: 4})) # Hits the cache and just returns the value. + func(5, Hash_adapter({5: 6, 7: 8})) # Also uses cache, since the Hash_adapter is ignored. + + In the above function, the "b" value is one that has been wrapped with Hash_adappter, and + therefore "b.actual_value" is necessary to access the original value. + ''' + + def __init__(self, actual_value): + self.actual_value = actual_value + + def __eq__(self, other): + return True + + def __hash__(self): + return 0 + + +UNHASHABLE_TYPES = (dict, list, set) + + +def cache_ignoring_unhashable_arguments(function): + ''' + A function decorator that caches calls to the decorated function but ignores any unhashable + arguments when performing cache lookups. This is intended to be a drop-in replacement for + functools.cache. + + Example usage: + + @cache_ignoring_unhashable_arguments + def func(a, b): + print(a, b) + return a + + func(5, {1: 2, 3: 4}) # Calls func(), prints, and returns. + func(5, {1: 2, 3: 4}) # Hits the cache and just returns the value. + func(5, {5: 6, 7: 8}) # Also uses cache, since the unhashable value (the dict) is ignored. + ''' + + @functools.cache + def cached_function(*args, **kwargs): + return function( + *(arg.actual_value if isinstance(arg, Hash_adapter) else arg for arg in args), + **{ + key: value.actual_value if isinstance(value, Hash_adapter) else value + for (key, value) in kwargs.items() + }, + ) + + @functools.wraps(function) + def wrapper_function(*args, **kwargs): + return cached_function( + *(Hash_adapter(arg) if isinstance(arg, UNHASHABLE_TYPES) else arg for arg in args), + **{ + key: Hash_adapter(value) if isinstance(value, UNHASHABLE_TYPES) else value + for (key, value) in kwargs.items() + }, + ) + + wrapper_function.cache_clear = cached_function.cache_clear + + return wrapper_function + + CREDENTIAL_PATTERN = re.compile(r'\{credential( +(?P.*))?\}') -@functools.cache -def resolve_credential(value): +@cache_ignoring_unhashable_arguments +def resolve_credential(value, config): ''' - Given a configuration value containing a string like "{credential hookname credentialname}", resolve it by - calling the relevant hook to get the actual credential value. If the given value does not - actually contain a credential tag, then return it unchanged. + Given a configuration value containing a string like "{credential hookname credentialname}" and + a configuration dict, resolve the credential by calling the relevant hook to get the actual + credential value. If the given value does not actually contain a credential tag, then return it + unchanged. - Cache the value so repeated calls to this function don't need to load the credential repeatedly. + Cache the value (ignoring the config for purposes of caching), so repeated calls to this + function don't need to load the credential repeatedly. Raise ValueError if the config could not be parsed or the credential could not be loaded. ''' @@ -40,5 +120,5 @@ def resolve_credential(value): raise ValueError(f'Cannot load credential with invalid syntax "{value}"') return borgmatic.hooks.dispatch.call_hook( - 'load_credential', {}, hook_name, tuple(credential_parameters) + 'load_credential', config, hook_name, tuple(credential_parameters) ) diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index 11e3eddc..67d50d49 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -26,11 +26,11 @@ def make_dump_path(base_directory): # pragma: no cover SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys') -def database_names_to_dump(database, extra_environment, dry_run): +def database_names_to_dump(database, config, extra_environment, dry_run): ''' - Given a requested database config, return the corresponding sequence of database names to dump. - In the case of "all", query for the names of databases on the configured host and return them, - excluding any system databases that will cause problems during restore. + Given a requested database config and a configuration dict, return the corresponding sequence of + database names to dump. In the case of "all", query for the names of databases on the configured + host and return them, excluding any system databases that will cause problems during restore. ''' if database['name'] != 'all': return (database['name'],) @@ -47,7 +47,10 @@ def database_names_to_dump(database, extra_environment, dry_run): + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + ( - ('--user', borgmatic.hooks.credential.parse.resolve_credential(database['username'])) + ( + '--user', + borgmatic.hooks.credential.parse.resolve_credential(database['username'], config), + ) if 'username' in database else () ) @@ -67,7 +70,7 @@ def database_names_to_dump(database, extra_environment, dry_run): def execute_dump_command( - database, dump_path, database_names, extra_environment, dry_run, dry_run_label + database, config, dump_path, database_names, extra_environment, dry_run, dry_run_label ): ''' Kick off a dump for the given MariaDB database (provided as a configuration dict) to a named @@ -102,7 +105,10 @@ def execute_dump_command( + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + ( - ('--user', borgmatic.hooks.credential.parse.resolve_credential(database['username'])) + ( + '--user', + borgmatic.hooks.credential.parse.resolve_credential(database['username'], config), + ) if 'username' in database else () ) @@ -162,7 +168,11 @@ def dump_data_sources( for database in databases: dump_path = make_dump_path(borgmatic_runtime_directory) extra_environment = ( - {'MYSQL_PWD': borgmatic.hooks.credential.parse.resolve_credential(database['password'])} + { + 'MYSQL_PWD': borgmatic.hooks.credential.parse.resolve_credential( + database['password'], config + ) + } if 'password' in database else None ) @@ -181,6 +191,7 @@ def dump_data_sources( processes.append( execute_dump_command( renamed_database, + config, dump_path, (dump_name,), extra_environment, @@ -192,6 +203,7 @@ def dump_data_sources( processes.append( execute_dump_command( database, + config, dump_path, dump_database_names, extra_environment, @@ -265,12 +277,18 @@ def restore_data_source_dump( connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')) ) username = borgmatic.hooks.credential.parse.resolve_credential( - connection_params['username'] - or data_source.get('restore_username', data_source.get('username')) + ( + connection_params['username'] + or data_source.get('restore_username', data_source.get('username')) + ), + config, ) password = borgmatic.hooks.credential.parse.resolve_credential( - connection_params['password'] - or data_source.get('restore_password', data_source.get('password')) + ( + connection_params['password'] + or data_source.get('restore_password', data_source.get('password')) + ), + config, ) mariadb_restore_command = tuple( diff --git a/borgmatic/hooks/data_source/mongodb.py b/borgmatic/hooks/data_source/mongodb.py index 8015d35f..2148f009 100644 --- a/borgmatic/hooks/data_source/mongodb.py +++ b/borgmatic/hooks/data_source/mongodb.py @@ -69,7 +69,7 @@ def dump_data_sources( if dry_run: continue - command = build_dump_command(database, dump_filename, dump_format) + command = build_dump_command(database, config, dump_filename, dump_format) if dump_format == 'directory': dump.create_parent_directory_for_dump(dump_filename) @@ -88,7 +88,7 @@ def dump_data_sources( return processes -def build_dump_command(database, dump_filename, dump_format): +def build_dump_command(database, config, dump_filename, dump_format): ''' Return the mongodump command from a single database configuration. ''' @@ -103,7 +103,9 @@ def build_dump_command(database, dump_filename, dump_format): ( '--username', shlex.quote( - borgmatic.hooks.credential.parse.resolve_credential(database['username']) + borgmatic.hooks.credential.parse.resolve_credential( + database['username'], config + ) ), ) if 'username' in database @@ -113,7 +115,9 @@ def build_dump_command(database, dump_filename, dump_format): ( '--password', shlex.quote( - borgmatic.hooks.credential.parse.resolve_credential(database['password']) + borgmatic.hooks.credential.parse.resolve_credential( + database['password'], config + ) ), ) if 'password' in database @@ -192,7 +196,7 @@ def restore_data_source_dump( data_source.get('hostname'), ) restore_command = build_restore_command( - extract_process, data_source, dump_filename, connection_params + extract_process, data_source, config, dump_filename, connection_params ) logger.debug(f"Restoring MongoDB database {data_source['name']}{dry_run_label}") @@ -209,7 +213,7 @@ def restore_data_source_dump( ) -def build_restore_command(extract_process, database, dump_filename, connection_params): +def build_restore_command(extract_process, database, config, dump_filename, connection_params): ''' Return the mongorestore command from a single database configuration. ''' @@ -218,10 +222,18 @@ def build_restore_command(extract_process, database, dump_filename, connection_p ) port = str(connection_params['port'] or database.get('restore_port', database.get('port', ''))) username = borgmatic.hooks.credential.parse.resolve_credential( - connection_params['username'] or database.get('restore_username', database.get('username')) + ( + connection_params['username'] + or database.get('restore_username', database.get('username')) + ), + config, ) password = borgmatic.hooks.credential.parse.resolve_credential( - connection_params['password'] or database.get('restore_password', database.get('password')) + ( + connection_params['password'] + or database.get('restore_password', database.get('password')) + ), + config, ) command = ['mongorestore'] diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index 4b9fd320..aa5a954e 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -26,11 +26,11 @@ def make_dump_path(base_directory): # pragma: no cover SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys') -def database_names_to_dump(database, extra_environment, dry_run): +def database_names_to_dump(database, config, extra_environment, dry_run): ''' - Given a requested database config, return the corresponding sequence of database names to dump. - In the case of "all", query for the names of databases on the configured host and return them, - excluding any system databases that will cause problems during restore. + Given a requested database config and a configuration dict, return the corresponding sequence of + database names to dump. In the case of "all", query for the names of databases on the configured + host and return them, excluding any system databases that will cause problems during restore. ''' if database['name'] != 'all': return (database['name'],) @@ -47,7 +47,10 @@ def database_names_to_dump(database, extra_environment, dry_run): + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + ( - ('--user', borgmatic.hooks.credential.parse.resolve_credential(database['username'])) + ( + '--user', + borgmatic.hooks.credential.parse.resolve_credential(database['username'], config), + ) if 'username' in database else () ) @@ -67,7 +70,7 @@ def database_names_to_dump(database, extra_environment, dry_run): def execute_dump_command( - database, dump_path, database_names, extra_environment, dry_run, dry_run_label + database, config, dump_path, database_names, extra_environment, dry_run, dry_run_label ): ''' Kick off a dump for the given MySQL/MariaDB database (provided as a configuration dict) to a @@ -101,7 +104,10 @@ def execute_dump_command( + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + ( - ('--user', borgmatic.hooks.credential.parse.resolve_credential(database['username'])) + ( + '--user', + borgmatic.hooks.credential.parse.resolve_credential(database['username'], config), + ) if 'username' in database else () ) @@ -161,11 +167,15 @@ def dump_data_sources( for database in databases: dump_path = make_dump_path(borgmatic_runtime_directory) extra_environment = ( - {'MYSQL_PWD': borgmatic.hooks.credential.parse.resolve_credential(database['password'])} + { + 'MYSQL_PWD': borgmatic.hooks.credential.parse.resolve_credential( + database['password'], config + ) + } if 'password' in database else None ) - dump_database_names = database_names_to_dump(database, extra_environment, dry_run) + dump_database_names = database_names_to_dump(database, config, extra_environment, dry_run) if not dump_database_names: if dry_run: @@ -180,6 +190,7 @@ def dump_data_sources( processes.append( execute_dump_command( renamed_database, + config, dump_path, (dump_name,), extra_environment, @@ -191,6 +202,7 @@ def dump_data_sources( processes.append( execute_dump_command( database, + config, dump_path, dump_database_names, extra_environment, @@ -264,12 +276,18 @@ def restore_data_source_dump( connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')) ) username = borgmatic.hooks.credential.parse.resolve_credential( - connection_params['username'] - or data_source.get('restore_username', data_source.get('username')) + ( + connection_params['username'] + or data_source.get('restore_username', data_source.get('username')) + ), + config, ) password = borgmatic.hooks.credential.parse.resolve_credential( - connection_params['password'] - or data_source.get('restore_password', data_source.get('password')) + ( + connection_params['password'] + or data_source.get('restore_password', data_source.get('password')) + ), + config, ) mysql_restore_command = tuple( diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index 82678286..a9eafea0 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -25,7 +25,7 @@ def make_dump_path(base_directory): # pragma: no cover return dump.make_data_source_dump_path(base_directory, 'postgresql_databases') -def make_extra_environment(database, restore_connection_params=None): +def make_extra_environment(database, config, restore_connection_params=None): ''' Make the extra_environment dict from the given database configuration. If restore connection params are given, this is for a restore operation. @@ -35,12 +35,15 @@ def make_extra_environment(database, restore_connection_params=None): try: if restore_connection_params: extra['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential( - restore_connection_params.get('password') - or database.get('restore_password', database['password']) + ( + restore_connection_params.get('password') + or database.get('restore_password', database['password']) + ), + config, ) else: extra['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential( - database['password'] + database['password'], config ) except (AttributeError, KeyError): pass @@ -62,12 +65,12 @@ def make_extra_environment(database, restore_connection_params=None): EXCLUDED_DATABASE_NAMES = ('template0', 'template1') -def database_names_to_dump(database, extra_environment, dry_run): +def database_names_to_dump(database, config, extra_environment, dry_run): ''' - Given a requested database config, return the corresponding sequence of database names to dump. - In the case of "all" when a database format is given, query for the names of databases on the - configured host and return them. For "all" without a database format, just return a sequence - containing "all". + Given a requested database config and a configuration dict, return the corresponding sequence of + database names to dump. In the case of "all" when a database format is given, query for the + names of databases on the configured host and return them. For "all" without a database format, + just return a sequence containing "all". ''' requested_name = database['name'] @@ -89,7 +92,7 @@ def database_names_to_dump(database, extra_environment, dry_run): + ( ( '--username', - borgmatic.hooks.credential.parse.resolve_credential(database['username']), + borgmatic.hooks.credential.parse.resolve_credential(database['username'], config), ) if 'username' in database else () @@ -146,9 +149,9 @@ def dump_data_sources( logger.info(f'Dumping PostgreSQL databases{dry_run_label}') for database in databases: - extra_environment = make_extra_environment(database) + extra_environment = make_extra_environment(database, config) dump_path = make_dump_path(borgmatic_runtime_directory) - dump_database_names = database_names_to_dump(database, extra_environment, dry_run) + dump_database_names = database_names_to_dump(database, config, extra_environment, dry_run) if not dump_database_names: if dry_run: @@ -189,7 +192,7 @@ def dump_data_sources( '--username', shlex.quote( borgmatic.hooks.credential.parse.resolve_credential( - database['username'] + database['username'], config ) ), ) @@ -309,8 +312,11 @@ def restore_data_source_dump( connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')) ) username = borgmatic.hooks.credential.parse.resolve_credential( - connection_params['username'] - or data_source.get('restore_username', data_source.get('username')) + ( + connection_params['username'] + or data_source.get('restore_username', data_source.get('username')) + ), + config, ) all_databases = bool(data_source['name'] == 'all') @@ -364,7 +370,7 @@ def restore_data_source_dump( ) extra_environment = make_extra_environment( - data_source, restore_connection_params=connection_params + data_source, config, restore_connection_params=connection_params ) logger.debug(f"Restoring PostgreSQL database {data_source['name']}{dry_run_label}") diff --git a/borgmatic/hooks/monitoring/ntfy.py b/borgmatic/hooks/monitoring/ntfy.py index fb4adc14..9c479cdd 100644 --- a/borgmatic/hooks/monitoring/ntfy.py +++ b/borgmatic/hooks/monitoring/ntfy.py @@ -51,13 +51,13 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev try: username = borgmatic.hooks.credential.parse.resolve_credential( - hook_config.get('username') + hook_config.get('username'), config ) password = borgmatic.hooks.credential.parse.resolve_credential( - hook_config.get('password') + hook_config.get('password'), config ) access_token = borgmatic.hooks.credential.parse.resolve_credential( - hook_config.get('access_token') + hook_config.get('access_token'), config ) except ValueError as error: logger.warning(f'Ntfy credential error: {error}') diff --git a/borgmatic/hooks/monitoring/pagerduty.py b/borgmatic/hooks/monitoring/pagerduty.py index d7a54e62..e238957b 100644 --- a/borgmatic/hooks/monitoring/pagerduty.py +++ b/borgmatic/hooks/monitoring/pagerduty.py @@ -42,7 +42,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev try: integration_key = borgmatic.hooks.credential.parse.resolve_credential( - hook_config.get('integration_key') + hook_config.get('integration_key'), config ) except ValueError as error: logger.warning(f'PagerDuty credential error: {error}') diff --git a/borgmatic/hooks/monitoring/pushover.py b/borgmatic/hooks/monitoring/pushover.py index 61afdb98..86a1ac26 100644 --- a/borgmatic/hooks/monitoring/pushover.py +++ b/borgmatic/hooks/monitoring/pushover.py @@ -35,8 +35,10 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev state_config = hook_config.get(state.name.lower(), {}) try: - token = borgmatic.hooks.credential.parse.resolve_credential(hook_config.get('token')) - user = borgmatic.hooks.credential.parse.resolve_credential(hook_config.get('user')) + token = borgmatic.hooks.credential.parse.resolve_credential( + hook_config.get('token'), config + ) + user = borgmatic.hooks.credential.parse.resolve_credential(hook_config.get('user'), config) except ValueError as error: logger.warning(f'Pushover credential error: {error}') return diff --git a/borgmatic/hooks/monitoring/zabbix.py b/borgmatic/hooks/monitoring/zabbix.py index df8bfed9..54fcc196 100644 --- a/borgmatic/hooks/monitoring/zabbix.py +++ b/borgmatic/hooks/monitoring/zabbix.py @@ -37,9 +37,15 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev ) try: - username = borgmatic.hooks.credential.parse.resolve_credential(hook_config.get('username')) - password = borgmatic.hooks.credential.parse.resolve_credential(hook_config.get('password')) - api_key = borgmatic.hooks.credential.parse.resolve_credential(hook_config.get('api_key')) + username = borgmatic.hooks.credential.parse.resolve_credential( + hook_config.get('username'), config + ) + password = borgmatic.hooks.credential.parse.resolve_credential( + hook_config.get('password'), config + ) + api_key = borgmatic.hooks.credential.parse.resolve_credential( + hook_config.get('api_key'), config + ) except ValueError as error: logger.warning(f'Zabbix credential error: {error}') return diff --git a/tests/unit/borg/test_environment.py b/tests/unit/borg/test_environment.py index f118d3b1..a63c3470 100644 --- a/tests/unit/borg/test_environment.py +++ b/tests/unit/borg/test_environment.py @@ -26,7 +26,7 @@ def test_make_environment_with_passphrase_should_set_environment(): flexmock(module.os.environ).should_receive('get').and_return(None) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) environment = module.make_environment({'encryption_passphrase': 'pass'}) @@ -34,16 +34,17 @@ def test_make_environment_with_passphrase_should_set_environment(): def test_make_environment_with_credential_tag_passphrase_should_load_it_and_set_environment(): + config = {'encryption_passphrase': '{credential systemd pass}'} flexmock(module.borgmatic.borg.passcommand).should_receive( 'get_passphrase_from_passcommand' ).and_return(None) flexmock(module.os).should_receive('pipe').never() flexmock(module.os.environ).should_receive('get').and_return(None) flexmock(module.borgmatic.hooks.credential.parse).should_receive( - 'resolve_credential' - ).with_args('{credential systemd pass}').and_return('pass') + 'resolve_credential', + ).with_args('{credential systemd pass}', config).and_return('pass') - environment = module.make_environment({'encryption_passphrase': '{credential systemd pass}'}) + environment = module.make_environment(config) assert environment.get('BORG_PASSPHRASE') == 'pass' diff --git a/tests/unit/hooks/credential/test_parse.py b/tests/unit/hooks/credential/test_parse.py index 5fd4301a..c1c9e1b4 100644 --- a/tests/unit/hooks/credential/test_parse.py +++ b/tests/unit/hooks/credential/test_parse.py @@ -4,18 +4,49 @@ from flexmock import flexmock from borgmatic.hooks.credential import parse as module +def test_hash_adapter_is_always_equal(): + assert module.Hash_adapter({1: 2}) == module.Hash_adapter({3: 4}) + + +def test_hash_adapter_alwaysh_hashes_the_same(): + assert hash(module.Hash_adapter({1: 2})) == hash(module.Hash_adapter({3: 4})) + + +def test_cache_ignoring_unhashable_arguments_caches_arguments_after_first_call(): + hashable = 3 + unhashable = {1, 2} + calls = 0 + + @module.cache_ignoring_unhashable_arguments + def function(first, second, third): + nonlocal calls + calls += 1 + + assert first == hashable + assert second == unhashable + assert third == unhashable + + return first + + assert function(hashable, unhashable, third=unhashable) == hashable + assert calls == 1 + + assert function(hashable, unhashable, third=unhashable) == hashable + assert calls == 1 + + def test_resolve_credential_passes_through_string_without_credential(): module.resolve_credential.cache_clear() flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').never() - assert module.resolve_credential('{no credentials here}') == '{no credentials here}' + assert module.resolve_credential('{no credentials here}', config={}) == '{no credentials here}' def test_resolve_credential_passes_through_none(): module.resolve_credential.cache_clear() flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').never() - assert module.resolve_credential(None) is None + assert module.resolve_credential(None, config={}) is None @pytest.mark.parametrize('invalid_value', ('{credential}', '{credential }', '{credential systemd}')) @@ -24,7 +55,7 @@ def test_resolve_credential_with_invalid_credential_raises(invalid_value): flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').never() with pytest.raises(ValueError): - module.resolve_credential(invalid_value) + module.resolve_credential(invalid_value, config={}) def test_resolve_credential_with_valid_credential_loads_credential(): @@ -36,7 +67,7 @@ def test_resolve_credential_with_valid_credential_loads_credential(): ('mycredential',), ).and_return('result').once() - assert module.resolve_credential('{credential systemd mycredential}') == 'result' + assert module.resolve_credential('{credential systemd mycredential}', config={}) == 'result' def test_resolve_credential_with_valid_credential_and_quoted_parameters_loads_credential(): @@ -48,7 +79,7 @@ def test_resolve_credential_with_valid_credential_and_quoted_parameters_loads_cr ('my credential',), ).and_return('result').once() - assert module.resolve_credential('{credential systemd "my credential"}') == 'result' + assert module.resolve_credential('{credential systemd "my credential"}', config={}) == 'result' def test_resolve_credential_caches_credential_after_first_call(): @@ -60,5 +91,5 @@ def test_resolve_credential_caches_credential_after_first_call(): ('mycredential',), ).and_return('result').once() - assert module.resolve_credential('{credential systemd mycredential}') == 'result' - assert module.resolve_credential('{credential systemd mycredential}') == 'result' + assert module.resolve_credential('{credential systemd mycredential}', config={}) == 'result' + assert module.resolve_credential('{credential systemd mycredential}', config={}) == 'result' diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py index e55af230..30209f51 100644 --- a/tests/unit/hooks/data_source/test_mariadb.py +++ b/tests/unit/hooks/data_source/test_mariadb.py @@ -9,7 +9,7 @@ from borgmatic.hooks.data_source import mariadb as module def test_database_names_to_dump_passes_through_name(): extra_environment = flexmock() - names = module.database_names_to_dump({'name': 'foo'}, extra_environment, dry_run=False) + names = module.database_names_to_dump({'name': 'foo'}, {}, extra_environment, dry_run=False) assert names == ('foo',) @@ -18,7 +18,7 @@ def test_database_names_to_dump_bails_for_dry_run(): extra_environment = flexmock() flexmock(module).should_receive('execute_command_and_capture_output').never() - names = module.database_names_to_dump({'name': 'all'}, extra_environment, dry_run=True) + names = module.database_names_to_dump({'name': 'all'}, {}, extra_environment, dry_run=True) assert names == () @@ -27,13 +27,13 @@ def test_database_names_to_dump_queries_mariadb_for_database_names(): extra_environment = flexmock() flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('mariadb', '--skip-column-names', '--batch', '--execute', 'show schemas'), extra_environment=extra_environment, ).and_return('foo\nbar\nmysql\n').once() - names = module.database_names_to_dump({'name': 'all'}, extra_environment, dry_run=False) + names = module.database_names_to_dump({'name': 'all'}, {}, extra_environment, dry_run=False) assert names == ('foo', 'bar') @@ -55,7 +55,7 @@ def test_dump_data_sources_dumps_each_database(): flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( ('bar',) ) @@ -63,6 +63,7 @@ def test_dump_data_sources_dumps_each_database(): for name, process in zip(('foo', 'bar'), processes): flexmock(module).should_receive('execute_dump_command').with_args( database={'name': name}, + config={}, dump_path=object, database_names=(name,), extra_environment=object, @@ -89,13 +90,14 @@ def test_dump_data_sources_dumps_with_password(): flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( ('bar',) ) flexmock(module).should_receive('execute_dump_command').with_args( database=database, + config={}, dump_path=object, database_names=('foo',), extra_environment={'MYSQL_PWD': 'trustsome1'}, @@ -119,10 +121,11 @@ def test_dump_data_sources_dumps_all_databases_at_once(): flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) flexmock(module).should_receive('execute_dump_command').with_args( database={'name': 'all'}, + config={}, dump_path=object, database_names=('foo', 'bar'), extra_environment=object, @@ -146,12 +149,13 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) for name, process in zip(('foo', 'bar'), processes): flexmock(module).should_receive('execute_dump_command').with_args( database={'name': name, 'format': 'sql'}, + config={}, dump_path=object, database_names=(name,), extra_environment=object, @@ -186,7 +190,7 @@ def test_database_names_to_dump_runs_mariadb_with_list_options(): extra_environment=None, ).and_return(('foo\nbar')).once() - assert module.database_names_to_dump(database, None, '') == ('foo', 'bar') + assert module.database_names_to_dump(database, {}, None, '') == ('foo', 'bar') def test_database_names_to_dump_runs_non_default_mariadb_with_list_options(): @@ -207,7 +211,7 @@ def test_database_names_to_dump_runs_non_default_mariadb_with_list_options(): ), ).and_return(('foo\nbar')).once() - assert module.database_names_to_dump(database, None, '') == ('foo', 'bar') + assert module.database_names_to_dump(database, {}, None, '') == ('foo', 'bar') def test_execute_dump_command_runs_mariadb_dump(): @@ -216,7 +220,7 @@ def test_execute_dump_command_runs_mariadb_dump(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -235,6 +239,7 @@ def test_execute_dump_command_runs_mariadb_dump(): assert ( module.execute_dump_command( database={'name': 'foo'}, + config={}, dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -251,7 +256,7 @@ def test_execute_dump_command_runs_mariadb_dump_without_add_drop_database(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -269,6 +274,7 @@ def test_execute_dump_command_runs_mariadb_dump_without_add_drop_database(): assert ( module.execute_dump_command( database={'name': 'foo', 'add_drop_database': False}, + config={}, dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -285,7 +291,7 @@ def test_execute_dump_command_runs_mariadb_dump_with_hostname_and_port(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -310,6 +316,7 @@ def test_execute_dump_command_runs_mariadb_dump_with_hostname_and_port(): assert ( module.execute_dump_command( database={'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}, + config={}, dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -326,7 +333,7 @@ def test_execute_dump_command_runs_mariadb_dump_with_username_and_password(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -347,6 +354,7 @@ def test_execute_dump_command_runs_mariadb_dump_with_username_and_password(): assert ( module.execute_dump_command( database={'name': 'foo', 'username': 'root', 'password': 'trustsome1'}, + config={}, dump_path=flexmock(), database_names=('foo',), extra_environment={'MYSQL_PWD': 'trustsome1'}, @@ -363,7 +371,7 @@ def test_execute_dump_command_runs_mariadb_dump_with_options(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -383,6 +391,7 @@ def test_execute_dump_command_runs_mariadb_dump_with_options(): assert ( module.execute_dump_command( database={'name': 'foo', 'options': '--stuff=such'}, + config={}, dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -399,7 +408,7 @@ def test_execute_dump_command_runs_non_default_mariadb_dump_with_options(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -423,6 +432,7 @@ def test_execute_dump_command_runs_non_default_mariadb_dump_with_options(): 'mariadb_dump_command': 'custom_mariadb_dump', 'options': '--stuff=such', }, # Custom MariaDB dump command specified + config={}, dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -442,6 +452,7 @@ def test_execute_dump_command_with_duplicate_dump_skips_mariadb_dump(): assert ( module.execute_dump_command( database={'name': 'foo'}, + config={}, dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -457,7 +468,7 @@ def test_execute_dump_command_with_dry_run_skips_mariadb_dump(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').never() @@ -465,6 +476,7 @@ def test_execute_dump_command_with_dry_run_skips_mariadb_dump(): assert ( module.execute_dump_command( database={'name': 'foo'}, + config={}, dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -480,7 +492,7 @@ def test_dump_data_sources_errors_for_missing_all_databases(): flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( 'databases/localhost/all' ) @@ -502,7 +514,7 @@ def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( 'databases/localhost/all' ) @@ -527,7 +539,7 @@ def test_restore_data_source_dump_runs_mariadb_to_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mariadb', '--batch'), processes=[extract_process], @@ -558,7 +570,7 @@ def test_restore_data_source_dump_runs_mariadb_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mariadb', '--batch', '--harder'), processes=[extract_process], @@ -591,7 +603,7 @@ def test_restore_data_source_dump_runs_non_default_mariadb_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ('custom_mariadb', '--batch', '--harder'), processes=[extract_process], @@ -622,7 +634,7 @@ def test_restore_data_source_dump_runs_mariadb_with_hostname_and_port(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mariadb', @@ -662,7 +674,7 @@ def test_restore_data_source_dump_runs_mariadb_with_username_and_password(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mariadb', '--batch', '--user', 'root'), processes=[extract_process], @@ -703,7 +715,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mariadb', @@ -757,7 +769,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mariadb', @@ -798,7 +810,7 @@ def test_restore_data_source_dump_with_dry_run_skips_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').never() module.restore_data_source_dump( diff --git a/tests/unit/hooks/data_source/test_mongodb.py b/tests/unit/hooks/data_source/test_mongodb.py index 4603bfa9..ea22e824 100644 --- a/tests/unit/hooks/data_source/test_mongodb.py +++ b/tests/unit/hooks/data_source/test_mongodb.py @@ -126,7 +126,7 @@ def test_dump_data_sources_runs_mongodump_with_username_and_password(): ) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -247,9 +247,9 @@ def test_build_dump_command_with_username_injection_attack_gets_escaped(): database = {'name': 'test', 'username': 'bob; naughty-command'} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) - command = module.build_dump_command(database, dump_filename='test', dump_format='archive') + command = module.build_dump_command(database, {}, dump_filename='test', dump_format='archive') assert "'bob; naughty-command'" in command @@ -262,7 +262,7 @@ def test_restore_data_source_dump_runs_mongorestore(): flexmock(module.dump).should_receive('make_data_source_dump_filename') flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ['mongorestore', '--archive', '--drop'], processes=[extract_process], @@ -296,7 +296,7 @@ def test_restore_data_source_dump_runs_mongorestore_with_hostname_and_port(): flexmock(module.dump).should_receive('make_data_source_dump_filename') flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( [ 'mongorestore', @@ -344,7 +344,7 @@ def test_restore_data_source_dump_runs_mongorestore_with_username_and_password() flexmock(module.dump).should_receive('make_data_source_dump_filename') flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( [ 'mongorestore', @@ -398,7 +398,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ flexmock(module.dump).should_receive('make_data_source_dump_filename') flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( [ 'mongorestore', @@ -456,7 +456,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ flexmock(module.dump).should_receive('make_data_source_dump_filename') flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( [ 'mongorestore', @@ -502,7 +502,7 @@ def test_restore_data_source_dump_runs_mongorestore_with_options(): flexmock(module.dump).should_receive('make_data_source_dump_filename') flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ['mongorestore', '--archive', '--drop', '--harder'], processes=[extract_process], @@ -534,7 +534,7 @@ def test_restore_databases_dump_runs_mongorestore_with_schemas(): flexmock(module.dump).should_receive('make_data_source_dump_filename') flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( [ 'mongorestore', @@ -574,7 +574,7 @@ def test_restore_data_source_dump_runs_psql_for_all_database_dump(): flexmock(module.dump).should_receive('make_data_source_dump_filename') flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ['mongorestore', '--archive'], processes=[extract_process], @@ -605,7 +605,7 @@ def test_restore_data_source_dump_with_dry_run_skips_restore(): flexmock(module.dump).should_receive('make_data_source_dump_filename') flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').never() module.restore_data_source_dump( @@ -631,7 +631,7 @@ def test_restore_data_source_dump_without_extract_process_restores_from_disk(): flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('/dump/path') flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ['mongorestore', '--dir', '/dump/path', '--drop'], processes=[], diff --git a/tests/unit/hooks/data_source/test_mysql.py b/tests/unit/hooks/data_source/test_mysql.py index 252b23a5..3fcf43eb 100644 --- a/tests/unit/hooks/data_source/test_mysql.py +++ b/tests/unit/hooks/data_source/test_mysql.py @@ -9,7 +9,7 @@ from borgmatic.hooks.data_source import mysql as module def test_database_names_to_dump_passes_through_name(): extra_environment = flexmock() - names = module.database_names_to_dump({'name': 'foo'}, extra_environment, dry_run=False) + names = module.database_names_to_dump({'name': 'foo'}, {}, extra_environment, dry_run=False) assert names == ('foo',) @@ -18,10 +18,10 @@ def test_database_names_to_dump_bails_for_dry_run(): extra_environment = flexmock() flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_and_capture_output').never() - names = module.database_names_to_dump({'name': 'all'}, extra_environment, dry_run=True) + names = module.database_names_to_dump({'name': 'all'}, {}, extra_environment, dry_run=True) assert names == () @@ -30,13 +30,13 @@ def test_database_names_to_dump_queries_mysql_for_database_names(): extra_environment = flexmock() flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('mysql', '--skip-column-names', '--batch', '--execute', 'show schemas'), extra_environment=extra_environment, ).and_return('foo\nbar\nmysql\n').once() - names = module.database_names_to_dump({'name': 'all'}, extra_environment, dry_run=False) + names = module.database_names_to_dump({'name': 'all'}, {}, extra_environment, dry_run=False) assert names == ('foo', 'bar') @@ -63,6 +63,7 @@ def test_dump_data_sources_dumps_each_database(): for name, process in zip(('foo', 'bar'), processes): flexmock(module).should_receive('execute_dump_command').with_args( database={'name': name}, + config={}, dump_path=object, database_names=(name,), extra_environment=object, @@ -89,13 +90,14 @@ def test_dump_data_sources_dumps_with_password(): flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( ('bar',) ) flexmock(module).should_receive('execute_dump_command').with_args( database=database, + config={}, dump_path=object, database_names=('foo',), extra_environment={'MYSQL_PWD': 'trustsome1'}, @@ -120,6 +122,7 @@ def test_dump_data_sources_dumps_all_databases_at_once(): flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) flexmock(module).should_receive('execute_dump_command').with_args( database={'name': 'all'}, + config={}, dump_path=object, database_names=('foo', 'bar'), extra_environment=object, @@ -146,6 +149,7 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured for name, process in zip(('foo', 'bar'), processes): flexmock(module).should_receive('execute_dump_command').with_args( database={'name': name, 'format': 'sql'}, + config={}, dump_path=object, database_names=(name,), extra_environment=object, @@ -180,7 +184,7 @@ def test_database_names_to_dump_runs_mysql_with_list_options(): extra_environment=None, ).and_return(('foo\nbar')).once() - assert module.database_names_to_dump(database, None, '') == ('foo', 'bar') + assert module.database_names_to_dump(database, {}, None, '') == ('foo', 'bar') def test_database_names_to_dump_runs_non_default_mysql_with_list_options(): @@ -201,7 +205,7 @@ def test_database_names_to_dump_runs_non_default_mysql_with_list_options(): ), ).and_return(('foo\nbar')).once() - assert module.database_names_to_dump(database, None, '') == ('foo', 'bar') + assert module.database_names_to_dump(database, {}, None, '') == ('foo', 'bar') def test_execute_dump_command_runs_mysqldump(): @@ -210,7 +214,7 @@ def test_execute_dump_command_runs_mysqldump(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -229,6 +233,7 @@ def test_execute_dump_command_runs_mysqldump(): assert ( module.execute_dump_command( database={'name': 'foo'}, + config={}, dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -245,7 +250,7 @@ def test_execute_dump_command_runs_mysqldump_without_add_drop_database(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -263,6 +268,7 @@ def test_execute_dump_command_runs_mysqldump_without_add_drop_database(): assert ( module.execute_dump_command( database={'name': 'foo', 'add_drop_database': False}, + config={}, dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -279,7 +285,7 @@ def test_execute_dump_command_runs_mysqldump_with_hostname_and_port(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -304,6 +310,7 @@ def test_execute_dump_command_runs_mysqldump_with_hostname_and_port(): assert ( module.execute_dump_command( database={'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}, + config={}, dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -320,7 +327,7 @@ def test_execute_dump_command_runs_mysqldump_with_username_and_password(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -341,6 +348,7 @@ def test_execute_dump_command_runs_mysqldump_with_username_and_password(): assert ( module.execute_dump_command( database={'name': 'foo', 'username': 'root', 'password': 'trustsome1'}, + config={}, dump_path=flexmock(), database_names=('foo',), extra_environment={'MYSQL_PWD': 'trustsome1'}, @@ -357,7 +365,7 @@ def test_execute_dump_command_runs_mysqldump_with_options(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -377,6 +385,7 @@ def test_execute_dump_command_runs_mysqldump_with_options(): assert ( module.execute_dump_command( database={'name': 'foo', 'options': '--stuff=such'}, + config={}, dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -393,7 +402,7 @@ def test_execute_dump_command_runs_non_default_mysqldump(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -415,6 +424,7 @@ def test_execute_dump_command_runs_non_default_mysqldump(): 'name': 'foo', 'mysql_dump_command': 'custom_mysqldump', }, # Custom MySQL dump command specified + config={}, dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -434,6 +444,7 @@ def test_execute_dump_command_with_duplicate_dump_skips_mysqldump(): assert ( module.execute_dump_command( database={'name': 'foo'}, + config={}, dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -449,7 +460,7 @@ def test_execute_dump_command_with_dry_run_skips_mysqldump(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').never() @@ -457,6 +468,7 @@ def test_execute_dump_command_with_dry_run_skips_mysqldump(): assert ( module.execute_dump_command( database={'name': 'foo'}, + config={}, dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -472,7 +484,7 @@ def test_dump_data_sources_errors_for_missing_all_databases(): flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( 'databases/localhost/all' ) @@ -494,7 +506,7 @@ def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( 'databases/localhost/all' ) @@ -519,7 +531,7 @@ def test_restore_data_source_dump_runs_mysql_to_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mysql', '--batch'), processes=[extract_process], @@ -550,7 +562,7 @@ def test_restore_data_source_dump_runs_mysql_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mysql', '--batch', '--harder'), processes=[extract_process], @@ -581,7 +593,7 @@ def test_restore_data_source_dump_runs_non_default_mysql_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ('custom_mysql', '--batch', '--harder'), processes=[extract_process], @@ -612,7 +624,7 @@ def test_restore_data_source_dump_runs_mysql_with_hostname_and_port(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mysql', @@ -652,7 +664,7 @@ def test_restore_data_source_dump_runs_mysql_with_username_and_password(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mysql', '--batch', '--user', 'root'), processes=[extract_process], @@ -693,7 +705,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mysql', @@ -747,7 +759,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mysql', @@ -788,7 +800,7 @@ def test_restore_data_source_dump_with_dry_run_skips_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').never() module.restore_data_source_dump( diff --git a/tests/unit/hooks/data_source/test_postgresql.py b/tests/unit/hooks/data_source/test_postgresql.py index 3abd0740..6bdf083c 100644 --- a/tests/unit/hooks/data_source/test_postgresql.py +++ b/tests/unit/hooks/data_source/test_postgresql.py @@ -26,9 +26,9 @@ def test_make_extra_environment_maps_options_to_environment(): } flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) - extra_env = module.make_extra_environment(database) + extra_env = module.make_extra_environment(database, {}) assert extra_env == expected @@ -37,10 +37,10 @@ def test_make_extra_environment_with_cli_password_sets_correct_password(): database = {'name': 'foo', 'restore_password': 'trustsome1', 'password': 'anotherpassword'} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) extra = module.make_extra_environment( - database, restore_connection_params={'password': 'clipassword'} + database, {}, restore_connection_params={'password': 'clipassword'} ) assert extra['PGPASSWORD'] == 'clipassword' @@ -50,7 +50,7 @@ def test_make_extra_environment_without_cli_password_or_configured_password_does database = {'name': 'foo'} extra = module.make_extra_environment( - database, restore_connection_params={'username': 'someone'} + database, {}, restore_connection_params={'username': 'someone'} ) assert 'PGPASSWORD' not in extra @@ -59,7 +59,7 @@ def test_make_extra_environment_without_cli_password_or_configured_password_does def test_make_extra_environment_without_ssl_mode_does_not_set_ssl_mode(): database = {'name': 'foo'} - extra = module.make_extra_environment(database) + extra = module.make_extra_environment(database, {}) assert 'PGSSLMODE' not in extra @@ -67,41 +67,41 @@ def test_make_extra_environment_without_ssl_mode_does_not_set_ssl_mode(): def test_database_names_to_dump_passes_through_individual_database_name(): database = {'name': 'foo'} - assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ('foo',) + assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('foo',) def test_database_names_to_dump_passes_through_individual_database_name_with_format(): database = {'name': 'foo', 'format': 'custom'} - assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ('foo',) + assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('foo',) def test_database_names_to_dump_passes_through_all_without_format(): database = {'name': 'all'} - assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ('all',) + assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('all',) def test_database_names_to_dump_with_all_and_format_and_dry_run_bails(): database = {'name': 'all', 'format': 'custom'} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_and_capture_output').never() - assert module.database_names_to_dump(database, flexmock(), dry_run=True) == () + assert module.database_names_to_dump(database, {}, flexmock(), dry_run=True) == () def test_database_names_to_dump_with_all_and_format_lists_databases(): database = {'name': 'all', 'format': 'custom'} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_and_capture_output').and_return( 'foo,test,\nbar,test,"stuff and such"' ) - assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ( + assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ( 'foo', 'bar', ) @@ -111,7 +111,7 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostnam database = {'name': 'all', 'format': 'custom', 'hostname': 'localhost', 'port': 1234} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ( 'psql', @@ -128,7 +128,7 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostnam extra_environment=object, ).and_return('foo,test,\nbar,test,"stuff and such"') - assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ( + assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ( 'foo', 'bar', ) @@ -138,7 +138,7 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_usernam database = {'name': 'all', 'format': 'custom', 'username': 'postgres'} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ( 'psql', @@ -153,7 +153,7 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_usernam extra_environment=object, ).and_return('foo,test,\nbar,test,"stuff and such"') - assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ( + assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ( 'foo', 'bar', ) @@ -163,13 +163,13 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_options database = {'name': 'all', 'format': 'custom', 'list_options': '--harder'} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('psql', '--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only', '--harder'), extra_environment=object, ).and_return('foo,test,\nbar,test,"stuff and such"') - assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ( + assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ( 'foo', 'bar', ) @@ -179,12 +179,12 @@ def test_database_names_to_dump_with_all_and_format_excludes_particular_database database = {'name': 'all', 'format': 'custom'} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_and_capture_output').and_return( 'foo,test,\ntemplate0,test,blah' ) - assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ('foo',) + assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('foo',) def test_database_names_to_dump_with_all_and_psql_command_uses_custom_command(): @@ -195,7 +195,7 @@ def test_database_names_to_dump_with_all_and_psql_command_uses_custom_command(): } flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ( 'docker', @@ -213,7 +213,7 @@ def test_database_names_to_dump_with_all_and_psql_command_uses_custom_command(): extra_environment=object, ).and_return('foo,text').once() - assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ('foo',) + assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('foo',) def test_use_streaming_true_for_any_non_directory_format_databases(): @@ -248,7 +248,7 @@ def test_dump_data_sources_runs_pg_dump_for_each_database(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') for name, process in zip(('foo', 'bar'), processes): @@ -355,7 +355,7 @@ def test_dump_data_sources_with_dry_run_skips_pg_dump(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() @@ -384,7 +384,7 @@ def test_dump_data_sources_runs_pg_dump_with_hostname_and_port(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -432,7 +432,7 @@ def test_dump_data_sources_runs_pg_dump_with_username_and_password(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -478,7 +478,7 @@ def test_dump_data_sources_with_username_injection_attack_gets_escaped(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -521,7 +521,7 @@ def test_dump_data_sources_runs_pg_dump_with_directory_format(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_parent_directory_for_dump') flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() @@ -566,7 +566,7 @@ def test_dump_data_sources_runs_pg_dump_with_options(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -609,7 +609,7 @@ def test_dump_data_sources_runs_pg_dumpall_for_all_databases(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -641,7 +641,7 @@ def test_dump_data_sources_runs_non_default_pg_dump(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -679,7 +679,7 @@ def test_restore_data_source_dump_runs_pg_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') @@ -736,7 +736,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_hostname_and_port(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') @@ -801,7 +801,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_username_and_password(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('make_extra_environment').and_return( {'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'} ) @@ -875,7 +875,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('make_extra_environment').and_return( {'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'} ) @@ -957,7 +957,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('make_extra_environment').and_return( {'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'} ) @@ -1033,7 +1033,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') @@ -1090,7 +1090,7 @@ def test_restore_data_source_dump_runs_psql_for_all_database_dump(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') @@ -1132,7 +1132,7 @@ def test_restore_data_source_dump_runs_psql_for_plain_database_dump(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') @@ -1186,7 +1186,7 @@ def test_restore_data_source_dump_runs_non_default_pg_restore_and_psql(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') @@ -1250,7 +1250,7 @@ def test_restore_data_source_dump_with_dry_run_skips_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') @@ -1277,7 +1277,7 @@ def test_restore_data_source_dump_without_extract_process_restores_from_disk(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('/dump/path') @@ -1332,7 +1332,7 @@ def test_restore_data_source_dump_with_schemas_restores_schemas(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('/dump/path') diff --git a/tests/unit/hooks/monitoring/test_ntfy.py b/tests/unit/hooks/monitoring/test_ntfy.py index 3d8e21d4..d1a8c198 100644 --- a/tests/unit/hooks/monitoring/test_ntfy.py +++ b/tests/unit/hooks/monitoring/test_ntfy.py @@ -38,7 +38,7 @@ def test_ping_monitor_minimal_config_hits_hosted_ntfy_on_fail(): hook_config = {'topic': topic} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.FAIL), @@ -62,7 +62,7 @@ def test_ping_monitor_with_access_token_hits_hosted_ntfy_on_fail(): } flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.FAIL), @@ -88,7 +88,7 @@ def test_ping_monitor_with_username_password_and_access_token_ignores_username_p } flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.FAIL), @@ -114,7 +114,7 @@ def test_ping_monitor_with_username_password_hits_hosted_ntfy_on_fail(): } flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.FAIL), @@ -135,7 +135,7 @@ def test_ping_monitor_with_password_but_no_username_warns(): hook_config = {'topic': topic, 'password': 'fakepassword'} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.FAIL), @@ -157,7 +157,7 @@ def test_ping_monitor_with_username_but_no_password_warns(): hook_config = {'topic': topic, 'username': 'testuser'} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.FAIL), @@ -179,7 +179,7 @@ def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_start(): hook_config = {'topic': topic} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').never() module.ping_monitor( @@ -196,7 +196,7 @@ def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_finish(): hook_config = {'topic': topic} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').never() module.ping_monitor( @@ -213,7 +213,7 @@ def test_ping_monitor_minimal_config_hits_selfhosted_ntfy_on_fail(): hook_config = {'topic': topic, 'server': custom_base_url} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').with_args( f'{custom_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.FAIL), @@ -234,7 +234,7 @@ def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_fail_dry_run(): hook_config = {'topic': topic} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').never() module.ping_monitor( @@ -251,7 +251,7 @@ def test_ping_monitor_custom_message_hits_hosted_ntfy_on_fail(): hook_config = {'topic': topic, 'fail': custom_message_config} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=custom_message_headers, auth=None ).and_return(flexmock(ok=True)).once() @@ -270,7 +270,7 @@ def test_ping_monitor_custom_state_hits_hosted_ntfy_on_start(): hook_config = {'topic': topic, 'states': ['start', 'fail']} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.START), @@ -291,7 +291,7 @@ def test_ping_monitor_with_connection_error_logs_warning(): hook_config = {'topic': topic} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.FAIL), @@ -331,7 +331,7 @@ def test_ping_monitor_with_other_error_logs_warning(): hook_config = {'topic': topic} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) response = flexmock(ok=False) response.should_receive('raise_for_status').and_raise( module.requests.exceptions.RequestException diff --git a/tests/unit/hooks/monitoring/test_pagerduty.py b/tests/unit/hooks/monitoring/test_pagerduty.py index aff03f76..f7c73dde 100644 --- a/tests/unit/hooks/monitoring/test_pagerduty.py +++ b/tests/unit/hooks/monitoring/test_pagerduty.py @@ -6,7 +6,7 @@ from borgmatic.hooks.monitoring import pagerduty as module def test_ping_monitor_ignores_start_state(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').never() module.ping_monitor( @@ -22,7 +22,7 @@ def test_ping_monitor_ignores_start_state(): def test_ping_monitor_ignores_finish_state(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').never() module.ping_monitor( @@ -38,7 +38,7 @@ def test_ping_monitor_ignores_finish_state(): def test_ping_monitor_calls_api_for_fail_state(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').and_return(flexmock(ok=True)) module.ping_monitor( @@ -54,7 +54,7 @@ def test_ping_monitor_calls_api_for_fail_state(): def test_ping_monitor_dry_run_does_not_call_api(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').never() module.ping_monitor( @@ -70,7 +70,7 @@ def test_ping_monitor_dry_run_does_not_call_api(): def test_ping_monitor_with_connection_error_logs_warning(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').and_raise( module.requests.exceptions.ConnectionError ) @@ -107,7 +107,7 @@ def test_ping_monitor_with_other_error_logs_warning(): response = flexmock(ok=False) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) response.should_receive('raise_for_status').and_raise( module.requests.exceptions.RequestException ) diff --git a/tests/unit/hooks/monitoring/test_pushover.py b/tests/unit/hooks/monitoring/test_pushover.py index b10e5e14..be97994b 100644 --- a/tests/unit/hooks/monitoring/test_pushover.py +++ b/tests/unit/hooks/monitoring/test_pushover.py @@ -13,7 +13,7 @@ def test_ping_monitor_config_with_minimum_config_fail_state_backup_successfully_ hook_config = {'token': 'ksdjfwoweijfvwoeifvjmwghagy92', 'user': '983hfe0of902lkjfa2amanfgui'} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -43,7 +43,7 @@ def test_ping_monitor_config_with_minimum_config_start_state_backup_not_send_to_ hook_config = {'token': 'ksdjfwoweijfvwoeifvjmwghagy92', 'user': '983hfe0of902lkjfa2amanfgui'} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').never() @@ -71,7 +71,7 @@ def test_ping_monitor_start_state_backup_default_message_successfully_send_to_pu } flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -107,7 +107,7 @@ def test_ping_monitor_start_state_backup_custom_message_successfully_send_to_pus } flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -142,7 +142,7 @@ def test_ping_monitor_start_state_backup_default_message_with_priority_emergency } flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -180,7 +180,7 @@ def test_ping_monitor_start_state_backup_default_message_with_priority_emergency } flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -218,7 +218,7 @@ def test_ping_monitor_start_state_backup_default_message_with_priority_emergency } flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -259,7 +259,7 @@ def test_ping_monitor_start_state_backup_default_message_with_priority_high_decl flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').never() @@ -314,7 +314,7 @@ def test_ping_monitor_start_state_backup_based_on_documentation_advanced_example } flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -380,7 +380,7 @@ def test_ping_monitor_fail_state_backup_based_on_documentation_advanced_example_ } flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -451,7 +451,7 @@ def test_ping_monitor_finish_state_backup_based_on_documentation_advanced_exampl } flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -487,7 +487,7 @@ def test_ping_monitor_config_with_minimum_config_fail_state_backup_successfully_ hook_config = {'token': 'ksdjfwoweijfvwoeifvjmwghagy92', 'user': '983hfe0of902lkjfa2amanfgui'} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').and_return(flexmock(ok=True)).never() @@ -511,7 +511,7 @@ def test_ping_monitor_config_incorrect_state_exit_early(): } flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').and_return(flexmock(ok=True)).never() @@ -537,7 +537,7 @@ def test_ping_monitor_push_post_error_bails(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) push_response = flexmock(ok=False) push_response.should_receive('raise_for_status').and_raise( module.requests.ConnectionError diff --git a/tests/unit/hooks/monitoring/test_zabbix.py b/tests/unit/hooks/monitoring/test_zabbix.py index cba05ada..1b237810 100644 --- a/tests/unit/hooks/monitoring/test_zabbix.py +++ b/tests/unit/hooks/monitoring/test_zabbix.py @@ -77,7 +77,7 @@ def test_ping_monitor_config_with_api_key_only_bails(): hook_config = {'api_key': API_KEY} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() flexmock(module.requests).should_receive('post').never() @@ -97,7 +97,7 @@ def test_ping_monitor_config_with_host_only_bails(): hook_config = {'host': HOST} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() flexmock(module.requests).should_receive('post').never() @@ -117,7 +117,7 @@ def test_ping_monitor_config_with_key_only_bails(): hook_config = {'key': KEY} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() flexmock(module.requests).should_receive('post').never() @@ -137,7 +137,7 @@ def test_ping_monitor_config_with_server_only_bails(): hook_config = {'server': SERVER} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() flexmock(module.requests).should_receive('post').never() @@ -156,7 +156,7 @@ def test_ping_monitor_config_user_password_no_zabbix_data_bails(): hook_config = {'server': SERVER, 'username': USERNAME, 'password': PASSWORD} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() flexmock(module.requests).should_receive('post').never() @@ -175,7 +175,7 @@ def test_ping_monitor_config_api_key_no_zabbix_data_bails(): hook_config = {'server': SERVER, 'api_key': API_KEY} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() flexmock(module.requests).should_receive('post').never() @@ -195,7 +195,7 @@ def test_ping_monitor_config_itemid_no_auth_data_bails(): hook_config = {'server': SERVER, 'itemid': ITEMID} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() flexmock(module.requests).should_receive('post').never() @@ -215,7 +215,7 @@ def test_ping_monitor_config_host_and_key_no_auth_data_bails(): hook_config = {'server': SERVER, 'host': HOST, 'key': KEY} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() flexmock(module.requests).should_receive('post').never() @@ -235,7 +235,7 @@ def test_ping_monitor_config_host_and_key_with_api_key_auth_data_successful(): hook_config = {'server': SERVER, 'host': HOST, 'key': KEY, 'api_key': API_KEY} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').with_args( f'{SERVER}', headers=AUTH_HEADERS_API_KEY, @@ -257,7 +257,7 @@ def test_ping_monitor_config_host_and_missing_key_bails(): hook_config = {'server': SERVER, 'host': HOST, 'api_key': API_KEY} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() flexmock(module.requests).should_receive('post').never() @@ -275,7 +275,7 @@ def test_ping_monitor_config_key_and_missing_host_bails(): hook_config = {'server': SERVER, 'key': KEY, 'api_key': API_KEY} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() flexmock(module.requests).should_receive('post').never() @@ -302,7 +302,7 @@ def test_ping_monitor_config_host_and_key_with_username_password_auth_data_succe flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) auth_response = flexmock(ok=True) auth_response.should_receive('json').and_return( {'jsonrpc': '2.0', 'result': '3fe6ed01a69ebd79907a120bcd04e494', 'id': 1} @@ -343,7 +343,7 @@ def test_ping_monitor_config_host_and_key_with_username_password_auth_data_and_a flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) auth_response = flexmock(ok=False) auth_response.should_receive('json').and_return( {'jsonrpc': '2.0', 'result': '3fe6ed01a69ebd79907a120bcd04e494', 'id': 1} @@ -384,7 +384,7 @@ def test_ping_monitor_config_host_and_key_with_username_and_missing_password_bai flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() flexmock(module.requests).should_receive('post').never() @@ -408,7 +408,7 @@ def test_ping_monitor_config_host_and_key_with_password_and_missing_username_bai flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() flexmock(module.requests).should_receive('post').never() @@ -428,7 +428,7 @@ def test_ping_monitor_config_itemid_with_api_key_auth_data_successful(): hook_config = {'server': SERVER, 'itemid': ITEMID, 'api_key': API_KEY} flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').with_args( f'{SERVER}', headers=AUTH_HEADERS_API_KEY, @@ -453,7 +453,7 @@ def test_ping_monitor_config_itemid_with_username_password_auth_data_successful( flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) auth_response = flexmock(ok=True) auth_response.should_receive('json').and_return( {'jsonrpc': '2.0', 'result': '3fe6ed01a69ebd79907a120bcd04e494', 'id': 1} @@ -488,7 +488,7 @@ def test_ping_monitor_config_itemid_with_username_password_auth_data_and_push_po flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value: value) + ).replace_with(lambda value, config: value) auth_response = flexmock(ok=True) auth_response.should_receive('json').and_return( {'jsonrpc': '2.0', 'result': '3fe6ed01a69ebd79907a120bcd04e494', 'id': 1} From 2ca23b629c30c35cfbed8e2aaa48289ff6b0809c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 14 Feb 2025 15:33:30 -0800 Subject: [PATCH 025/226] Add end-to-end tests for new credential hooks, along with some related configuration options. --- borgmatic/config/schema.yaml | 22 ++++++ borgmatic/hooks/credential/container.py | 8 ++- borgmatic/hooks/credential/keepassxc.py | 5 +- borgmatic/hooks/data_source/mariadb.py | 2 +- docs/how-to/provide-your-passwords.md | 18 +++++ .../end-to-end/commands/fake_keepassxc_cli.py | 29 ++++++++ .../hooks/credential/test_container.py | 68 +++++++++++++++++++ .../end-to-end/hooks/credential/test_file.py | 68 +++++++++++++++++++ .../hooks/credential/test_keepassxc.py | 67 ++++++++++++++++++ .../hooks/credential/test_systemd.py | 6 +- tests/unit/hooks/credential/test_container.py | 15 ++++ tests/unit/hooks/credential/test_keepassxc.py | 44 +++++++++++- 12 files changed, 342 insertions(+), 10 deletions(-) create mode 100644 tests/end-to-end/commands/fake_keepassxc_cli.py create mode 100644 tests/end-to-end/hooks/credential/test_container.py create mode 100644 tests/end-to-end/hooks/credential/test_file.py create mode 100644 tests/end-to-end/hooks/credential/test_keepassxc.py diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index bc59edbc..1e238721 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -2402,3 +2402,25 @@ properties: description: | Configuration for integration with Linux LVM (Logical Volume Manager). + container: + type: object + additionalProperties: false + properties: + secrets_directory: + type: string + description: | + Secrets directory to use instead of "/run/secrets". + example: /path/to/secrets + description: | + Configuration for integration with Docker or Podman secrets. + keepassxc: + type: object + additionalProperties: false + properties: + keepassxc_cli_command: + type: string + description: | + Command to use instead of "keepassxc-cli". + example: /usr/local/bin/keepassxc-cli + description: | + Configuration for integration with the KeePassXC password manager. diff --git a/borgmatic/hooks/credential/container.py b/borgmatic/hooks/credential/container.py index 2bf976be..865d2092 100644 --- a/borgmatic/hooks/credential/container.py +++ b/borgmatic/hooks/credential/container.py @@ -6,7 +6,7 @@ logger = logging.getLogger(__name__) SECRET_NAME_PATTERN = re.compile(r'^\w+$') -SECRETS_DIRECTORY = '/run/secrets' +DEFAULT_SECRETS_DIRECTORY = '/run/secrets' def load_credential(hook_config, config, credential_parameters): @@ -27,7 +27,11 @@ def load_credential(hook_config, config, credential_parameters): raise ValueError(f'Cannot load invalid secret name: "{secret_name}"') try: - with open(os.path.join(SECRETS_DIRECTORY, secret_name)) as secret_file: + with open( + os.path.join( + (hook_config or {}).get('secrets_directory', DEFAULT_SECRETS_DIRECTORY), secret_name + ) + ) as secret_file: return secret_file.read().rstrip(os.linesep) except (FileNotFoundError, OSError) as error: logger.warning(error) diff --git a/borgmatic/hooks/credential/keepassxc.py b/borgmatic/hooks/credential/keepassxc.py index e12ef544..b280cc6a 100644 --- a/borgmatic/hooks/credential/keepassxc.py +++ b/borgmatic/hooks/credential/keepassxc.py @@ -1,5 +1,6 @@ import logging import os +import shlex import borgmatic.execute @@ -27,8 +28,8 @@ def load_credential(hook_config, config, credential_parameters): ) return borgmatic.execute.execute_command_and_capture_output( - ( - 'keepassxc-cli', + tuple(shlex.split((hook_config or {}).get('keepassxc_cli_command', 'keepassxc-cli'))) + + ( 'show', '--show-protected', '--attributes', diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index 67d50d49..47d1dfea 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -176,7 +176,7 @@ def dump_data_sources( if 'password' in database else None ) - dump_database_names = database_names_to_dump(database, extra_environment, dry_run) + dump_database_names = database_names_to_dump(database, config, extra_environment, dry_run) if not dump_database_names: if dry_run: diff --git a/docs/how-to/provide-your-passwords.md b/docs/how-to/provide-your-passwords.md index b4b5a8df..4fae07b9 100644 --- a/docs/how-to/provide-your-passwords.md +++ b/docs/how-to/provide-your-passwords.md @@ -191,6 +191,16 @@ For specifics about which options are supported, see the [configuration reference](https://torsion.org/borgmatic/docs/reference/configuration/). +You can also optionally override the `/run/secrets` directory that borgmatic reads secrets from +inside a container: + +```yaml +container: + secrets_directory: /path/to/secrets +``` + +But you should only need to do this for development or testing purposes. + ### KeePassXC passwords @@ -236,6 +246,14 @@ For specifics about which options are supported, see the [configuration reference](https://torsion.org/borgmatic/docs/reference/configuration/). +You can also optionally override the `keepassxc-cli` command that borgmatic calls to load +passwords: + +```yaml +keepassxc: + keepassxc_cli_command: /usr/local/bin/keepassxc-cli +``` + ### File-based credentials diff --git a/tests/end-to-end/commands/fake_keepassxc_cli.py b/tests/end-to-end/commands/fake_keepassxc_cli.py new file mode 100644 index 00000000..4c22c490 --- /dev/null +++ b/tests/end-to-end/commands/fake_keepassxc_cli.py @@ -0,0 +1,29 @@ +import argparse +import sys + + +def parse_arguments(*unparsed_arguments): + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument('command') + parser.add_argument('--show-protected', action='store_true') + parser.add_argument('--attributes') + parser.add_argument('database_path') + parser.add_argument('attribute_name') + + return parser.parse_args(unparsed_arguments) + + +def main(): + arguments = parse_arguments(*sys.argv[1:]) + + assert arguments.command == 'show' + assert arguments.show_protected + assert arguments.attributes == 'Password' + assert arguments.database_path.endswith('.kdbx') + assert arguments.attribute_name + + print('test') + + +if __name__ == '__main__': + main() diff --git a/tests/end-to-end/hooks/credential/test_container.py b/tests/end-to-end/hooks/credential/test_container.py new file mode 100644 index 00000000..6279f522 --- /dev/null +++ b/tests/end-to-end/hooks/credential/test_container.py @@ -0,0 +1,68 @@ +import json +import os +import shutil +import subprocess +import sys +import tempfile + + +def generate_configuration(config_path, repository_path, secrets_directory): + ''' + Generate borgmatic configuration into a file at the config path, and update the defaults so as + to work for testing, including updating the source directories, injecting the given repository + path, and tacking on an encryption passphrase loaded from container secrets in the given secrets + directory. + ''' + subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' ')) + config = ( + open(config_path) + .read() + .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path) + .replace('- path: /mnt/backup', '') + .replace('label: local', '') + .replace('- /home/user/path with spaces', '') + .replace('- /home', f'- {config_path}') + .replace('- /etc', '') + .replace('- /var/log/syslog*', '') + + '\nencryption_passphrase: "{credential container mysecret}"' + + f'\ncontainer:\n secrets_directory: {secrets_directory}' + ) + config_file = open(config_path, 'w') + config_file.write(config) + config_file.close() + + +def test_container_secret(): + # Create a Borg repository. + temporary_directory = tempfile.mkdtemp() + repository_path = os.path.join(temporary_directory, 'test.borg') + + original_working_directory = os.getcwd() + os.chdir(temporary_directory) + + try: + config_path = os.path.join(temporary_directory, 'test.yaml') + generate_configuration(config_path, repository_path, secrets_directory=temporary_directory) + + secret_path = os.path.join(temporary_directory, 'mysecret') + with open(secret_path, 'w') as secret_file: + secret_file.write('test') + + subprocess.check_call( + f'borgmatic -v 2 --config {config_path} repo-create --encryption repokey'.split(' '), + ) + + # Run borgmatic to generate a backup archive, and then list it to make sure it exists. + subprocess.check_call( + f'borgmatic --config {config_path}'.split(' '), + ) + output = subprocess.check_output( + f'borgmatic --config {config_path} list --json'.split(' '), + ).decode(sys.stdout.encoding) + parsed_output = json.loads(output) + + assert len(parsed_output) == 1 + assert len(parsed_output[0]['archives']) == 1 + finally: + os.chdir(original_working_directory) + shutil.rmtree(temporary_directory) diff --git a/tests/end-to-end/hooks/credential/test_file.py b/tests/end-to-end/hooks/credential/test_file.py new file mode 100644 index 00000000..0ae9b37a --- /dev/null +++ b/tests/end-to-end/hooks/credential/test_file.py @@ -0,0 +1,68 @@ +import json +import os +import shutil +import subprocess +import sys +import tempfile + + +def generate_configuration(config_path, repository_path, credential_path): + ''' + Generate borgmatic configuration into a file at the config path, and update the defaults so as + to work for testing, including updating the source directories, injecting the given repository + path, and tacking on an encryption passphrase loaded from file at the given credential path. + ''' + subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' ')) + config = ( + open(config_path) + .read() + .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path) + .replace('- path: /mnt/backup', '') + .replace('label: local', '') + .replace('- /home/user/path with spaces', '') + .replace('- /home', f'- {config_path}') + .replace('- /etc', '') + .replace('- /var/log/syslog*', '') + + '\nencryption_passphrase: "{credential file ' + + credential_path + + '}"' + ) + config_file = open(config_path, 'w') + config_file.write(config) + config_file.close() + + +def test_file_credential(): + # Create a Borg repository. + temporary_directory = tempfile.mkdtemp() + repository_path = os.path.join(temporary_directory, 'test.borg') + + original_working_directory = os.getcwd() + os.chdir(temporary_directory) + + try: + config_path = os.path.join(temporary_directory, 'test.yaml') + credential_path = os.path.join(temporary_directory, 'mycredential') + generate_configuration(config_path, repository_path, credential_path) + + with open(credential_path, 'w') as credential_file: + credential_file.write('test') + + subprocess.check_call( + f'borgmatic -v 2 --config {config_path} repo-create --encryption repokey'.split(' '), + ) + + # Run borgmatic to generate a backup archive, and then list it to make sure it exists. + subprocess.check_call( + f'borgmatic --config {config_path}'.split(' '), + ) + output = subprocess.check_output( + f'borgmatic --config {config_path} list --json'.split(' '), + ).decode(sys.stdout.encoding) + parsed_output = json.loads(output) + + assert len(parsed_output) == 1 + assert len(parsed_output[0]['archives']) == 1 + finally: + os.chdir(original_working_directory) + shutil.rmtree(temporary_directory) diff --git a/tests/end-to-end/hooks/credential/test_keepassxc.py b/tests/end-to-end/hooks/credential/test_keepassxc.py new file mode 100644 index 00000000..e990458f --- /dev/null +++ b/tests/end-to-end/hooks/credential/test_keepassxc.py @@ -0,0 +1,67 @@ +import json +import os +import shutil +import subprocess +import sys +import tempfile + + +def generate_configuration(config_path, repository_path): + ''' + Generate borgmatic configuration into a file at the config path, and update the defaults so as + to work for testing, including updating the source directories, injecting the given repository + path, and tacking on an encryption passphrase loaded from keepassxc-cli. + ''' + subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' ')) + config = ( + open(config_path) + .read() + .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path) + .replace('- path: /mnt/backup', '') + .replace('label: local', '') + .replace('- /home/user/path with spaces', '') + .replace('- /home', f'- {config_path}') + .replace('- /etc', '') + .replace('- /var/log/syslog*', '') + + '\nencryption_passphrase: "{credential keepassxc keys.kdbx mypassword}"' + + '\nkeepassxc:\n keepassxc_cli_command: python3 /app/tests/end-to-end/commands/fake_keepassxc_cli.py' + ) + config_file = open(config_path, 'w') + config_file.write(config) + config_file.close() + + +def test_keepassxc_password(): + # Create a Borg repository. + temporary_directory = tempfile.mkdtemp() + repository_path = os.path.join(temporary_directory, 'test.borg') + + original_working_directory = os.getcwd() + os.chdir(temporary_directory) + + try: + config_path = os.path.join(temporary_directory, 'test.yaml') + generate_configuration(config_path, repository_path) + + database_path = os.path.join(temporary_directory, 'keys.kdbx') + with open(database_path, 'w') as database_file: + database_file.write('fake KeePassXC database to pacify file existence check') + + subprocess.check_call( + f'borgmatic -v 2 --config {config_path} repo-create --encryption repokey'.split(' '), + ) + + # Run borgmatic to generate a backup archive, and then list it to make sure it exists. + subprocess.check_call( + f'borgmatic --config {config_path}'.split(' '), + ) + output = subprocess.check_output( + f'borgmatic --config {config_path} list --json'.split(' '), + ).decode(sys.stdout.encoding) + parsed_output = json.loads(output) + + assert len(parsed_output) == 1 + assert len(parsed_output[0]['archives']) == 1 + finally: + os.chdir(original_working_directory) + shutil.rmtree(temporary_directory) diff --git a/tests/end-to-end/hooks/credential/test_systemd.py b/tests/end-to-end/hooks/credential/test_systemd.py index d58302c3..9a263398 100644 --- a/tests/end-to-end/hooks/credential/test_systemd.py +++ b/tests/end-to-end/hooks/credential/test_systemd.py @@ -30,15 +30,13 @@ def generate_configuration(config_path, repository_path): config_file.close() -def test_borgmatic_command(): +def test_systemd_credential(): # Create a Borg repository. temporary_directory = tempfile.mkdtemp() repository_path = os.path.join(temporary_directory, 'test.borg') - extract_path = os.path.join(temporary_directory, 'extract') original_working_directory = os.getcwd() - os.mkdir(extract_path) - os.chdir(extract_path) + os.chdir(temporary_directory) try: config_path = os.path.join(temporary_directory, 'test.yaml') diff --git a/tests/unit/hooks/credential/test_container.py b/tests/unit/hooks/credential/test_container.py index 977f2f20..637d11fe 100644 --- a/tests/unit/hooks/credential/test_container.py +++ b/tests/unit/hooks/credential/test_container.py @@ -34,6 +34,21 @@ def test_load_credential_reads_named_secret_from_file(): ) +def test_load_credential_with_custom_secrets_directory_looks_there_for_secret_file(): + config = {'container': {'secrets_directory': '/secrets'}} + credential_stream = io.StringIO('password') + credential_stream.name = '/secrets/mysecret' + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('/secrets/mysecret').and_return(credential_stream) + + assert ( + module.load_credential( + hook_config=config['container'], config=config, credential_parameters=('mysecret',) + ) + == 'password' + ) + + def test_load_credential_with_file_not_found_error_raises(): builtins = flexmock(sys.modules['builtins']) builtins.should_receive('open').with_args('/run/secrets/mysecret').and_raise(FileNotFoundError) diff --git a/tests/unit/hooks/credential/test_keepassxc.py b/tests/unit/hooks/credential/test_keepassxc.py index f73e9de8..e25c8717 100644 --- a/tests/unit/hooks/credential/test_keepassxc.py +++ b/tests/unit/hooks/credential/test_keepassxc.py @@ -28,7 +28,19 @@ def test_load_credential_with_present_database_fetches_password_from_keepassxc() flexmock(module.os.path).should_receive('exists').and_return(True) flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' - ).and_return('password').once() + ).with_args( + ( + 'keepassxc-cli', + 'show', + '--show-protected', + '--attributes', + 'Password', + 'database.kdbx', + 'mypassword', + ) + ).and_return( + 'password' + ).once() assert ( module.load_credential( @@ -36,3 +48,33 @@ def test_load_credential_with_present_database_fetches_password_from_keepassxc() ) == 'password' ) + + +def test_load_credential_with_custom_keepassxc_cli_command_calls_it(): + config = {'keepassxc': {'keepassxc_cli_command': '/usr/local/bin/keepassxc-cli --some-option'}} + flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).with_args( + ( + '/usr/local/bin/keepassxc-cli', + '--some-option', + 'show', + '--show-protected', + '--attributes', + 'Password', + 'database.kdbx', + 'mypassword', + ) + ).and_return( + 'password' + ).once() + + assert ( + module.load_credential( + hook_config=config['keepassxc'], + config=config, + credential_parameters=('database.kdbx', 'mypassword'), + ) + == 'password' + ) From e02a0e632250164bd6b2f55b1fc53b805497faa6 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 14 Feb 2025 19:35:12 -0800 Subject: [PATCH 026/226] Support working directory for container and file credential hooks. --- borgmatic/hooks/credential/container.py | 4 +++- borgmatic/hooks/credential/file.py | 4 +++- tests/unit/hooks/credential/test_container.py | 17 +++++++++++++++++ tests/unit/hooks/credential/test_file.py | 18 ++++++++++++++++++ 4 files changed, 41 insertions(+), 2 deletions(-) diff --git a/borgmatic/hooks/credential/container.py b/borgmatic/hooks/credential/container.py index 865d2092..5b41409b 100644 --- a/borgmatic/hooks/credential/container.py +++ b/borgmatic/hooks/credential/container.py @@ -29,7 +29,9 @@ def load_credential(hook_config, config, credential_parameters): try: with open( os.path.join( - (hook_config or {}).get('secrets_directory', DEFAULT_SECRETS_DIRECTORY), secret_name + config.get('working_directory', ''), + (hook_config or {}).get('secrets_directory', DEFAULT_SECRETS_DIRECTORY), + secret_name, ) ) as secret_file: return secret_file.read().rstrip(os.linesep) diff --git a/borgmatic/hooks/credential/file.py b/borgmatic/hooks/credential/file.py index f9481954..387b31b2 100644 --- a/borgmatic/hooks/credential/file.py +++ b/borgmatic/hooks/credential/file.py @@ -18,7 +18,9 @@ def load_credential(hook_config, config, credential_parameters): raise ValueError(f'Cannot load invalid credential: "{' '.join(credential_parameters)}"') try: - with open(credential_path) as credential_file: + with open( + os.path.join(config.get('working_directory', ''), credential_path) + ) as credential_file: return credential_file.read().rstrip(os.linesep) except (FileNotFoundError, OSError) as error: logger.warning(error) diff --git a/tests/unit/hooks/credential/test_container.py b/tests/unit/hooks/credential/test_container.py index 637d11fe..5f03c871 100644 --- a/tests/unit/hooks/credential/test_container.py +++ b/tests/unit/hooks/credential/test_container.py @@ -49,6 +49,23 @@ def test_load_credential_with_custom_secrets_directory_looks_there_for_secret_fi ) +def test_load_credential_with_custom_secrets_directory_prefixes_it_with_working_directory(): + config = {'container': {'secrets_directory': 'secrets'}, 'working_directory': '/working'} + credential_stream = io.StringIO('password') + credential_stream.name = '/working/secrets/mysecret' + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('/working/secrets/mysecret').and_return( + credential_stream + ) + + assert ( + module.load_credential( + hook_config=config['container'], config=config, credential_parameters=('mysecret',) + ) + == 'password' + ) + + def test_load_credential_with_file_not_found_error_raises(): builtins = flexmock(sys.modules['builtins']) builtins.should_receive('open').with_args('/run/secrets/mysecret').and_raise(FileNotFoundError) diff --git a/tests/unit/hooks/credential/test_file.py b/tests/unit/hooks/credential/test_file.py index 219fcec7..aff10b89 100644 --- a/tests/unit/hooks/credential/test_file.py +++ b/tests/unit/hooks/credential/test_file.py @@ -38,6 +38,24 @@ def test_load_credential_reads_named_credential_from_file(): ) +def test_load_credential_reads_named_credential_from_file_using_working_directory(): + credential_stream = io.StringIO('password') + credential_stream.name = '/working/credentials/mycredential' + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('/working/credentials/mycredential').and_return( + credential_stream + ) + + assert ( + module.load_credential( + hook_config={}, + config={'working_directory': '/working'}, + credential_parameters=('credentials/mycredential',), + ) + == 'password' + ) + + def test_load_credential_with_file_not_found_error_raises(): builtins = flexmock(sys.modules['builtins']) builtins.should_receive('open').with_args('/credentials/mycredential').and_raise( From 056dfc6d335770b0d7ed6ec9307f0cc14d0562e4 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 15 Feb 2025 09:56:46 -0800 Subject: [PATCH 027/226] Add Btrfs "/" subvolume fix to NEWS. --- NEWS | 2 ++ borgmatic/hooks/data_source/btrfs.py | 4 +++- tests/unit/hooks/data_source/test_btrfs.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index e72c389e..68ac9ea6 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,8 @@ databases are enabled. * Add credential loading from file, KeePassXC, and Docker/Podman secrets. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/ + * Fix another error in the Btrfs hook when a subvolume mounted at "/" is configured in borgmatic's + source directories. 1.9.10 * #966: Add a "{credential ...}" syntax for loading systemd credentials into borgmatic diff --git a/borgmatic/hooks/data_source/btrfs.py b/borgmatic/hooks/data_source/btrfs.py index 99e75b26..2971aed8 100644 --- a/borgmatic/hooks/data_source/btrfs.py +++ b/borgmatic/hooks/data_source/btrfs.py @@ -299,8 +299,10 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d logger.debug(error) return - # Remove snapshot parent directory if it still exists (might not exist if snapshot was for '/') + # Remove the snapshot parent directory if it still exists. (It might not exist if the + # snapshot was for "/".) snapshot_parent_dir = snapshot_path.rsplit(subvolume.path, 1)[0] + if os.path.isdir(snapshot_parent_dir): shutil.rmtree(snapshot_parent_dir) diff --git a/tests/unit/hooks/data_source/test_btrfs.py b/tests/unit/hooks/data_source/test_btrfs.py index 8e7055c2..dde7b05b 100644 --- a/tests/unit/hooks/data_source/test_btrfs.py +++ b/tests/unit/hooks/data_source/test_btrfs.py @@ -860,7 +860,7 @@ def test_remove_data_source_dumps_with_delete_snapshot_called_process_error_bail ) -def test_remove_data_source_dumps_with_root_subvolume(): +def test_remove_data_source_dumps_with_root_subvolume_skips_duplicate_removal(): config = {'btrfs': {}} flexmock(module).should_receive('get_subvolumes').and_return( (module.Subvolume('/', contained_patterns=(Pattern('/etc'),)),) From 37ad398aff2607b910190da9d8d772fc2d1c732a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 16 Feb 2025 09:12:52 -0800 Subject: [PATCH 028/226] Add a ticket number to NEWS for (some of) the credential hook work. --- NEWS | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 68ac9ea6..b8370afa 100644 --- a/NEWS +++ b/NEWS @@ -1,8 +1,9 @@ 1.9.11.dev0 * #996: Fix the "create" action to omit the repository label prefix from Borg's output when databases are enabled. - * Add credential loading from file, KeePassXC, and Docker/Podman secrets. See the documentation for - more information: https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/ + * #795: Add credential loading from file, KeePassXC, and Docker/Podman secrets. See the + documentation for more information: + https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/ * Fix another error in the Btrfs hook when a subvolume mounted at "/" is configured in borgmatic's source directories. From 07ecc0ffd69515cade1c74e26c6587b8e97cd6fc Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 17 Feb 2025 11:03:36 -0800 Subject: [PATCH 029/226] Send the "encryption_passphrase" option to Borg via an anonymous pipe. --- NEWS | 2 + borgmatic/actions/check.py | 2 +- borgmatic/borg/borg.py | 2 +- borgmatic/borg/break_lock.py | 2 +- borgmatic/borg/change_passphrase.py | 2 +- borgmatic/borg/check.py | 2 +- borgmatic/borg/compact.py | 2 +- borgmatic/borg/create.py | 8 +- borgmatic/borg/delete.py | 2 +- borgmatic/borg/environment.py | 43 ++++-- borgmatic/borg/export_key.py | 2 +- borgmatic/borg/export_tar.py | 2 +- borgmatic/borg/extract.py | 8 +- borgmatic/borg/info.py | 4 +- borgmatic/borg/list.py | 6 +- borgmatic/borg/mount.py | 4 +- borgmatic/borg/passcommand.py | 16 +- borgmatic/borg/prune.py | 2 +- borgmatic/borg/repo_create.py | 2 +- borgmatic/borg/repo_delete.py | 2 +- borgmatic/borg/repo_info.py | 4 +- borgmatic/borg/repo_list.py | 6 +- borgmatic/borg/transfer.py | 2 +- borgmatic/borg/version.py | 2 +- borgmatic/execute.py | 74 ++++----- borgmatic/hooks/command.py | 14 +- borgmatic/hooks/data_source/mariadb.py | 39 ++--- borgmatic/hooks/data_source/mysql.py | 39 ++--- borgmatic/hooks/data_source/postgresql.py | 46 +++--- tests/unit/borg/test_borg.py | 30 ++-- tests/unit/borg/test_break_lock.py | 2 +- tests/unit/borg/test_change_passphrase.py | 2 +- tests/unit/borg/test_check.py | 14 +- tests/unit/borg/test_compact.py | 2 +- tests/unit/borg/test_create.py | 38 ++--- tests/unit/borg/test_delete.py | 6 +- tests/unit/borg/test_environment.py | 64 +++++--- tests/unit/borg/test_export_key.py | 2 +- tests/unit/borg/test_export_tar.py | 2 +- tests/unit/borg/test_extract.py | 8 +- tests/unit/borg/test_info.py | 4 +- tests/unit/borg/test_list.py | 24 +-- tests/unit/borg/test_mount.py | 6 +- tests/unit/borg/test_passcommand.py | 69 ++------- tests/unit/borg/test_prune.py | 4 +- tests/unit/borg/test_repo_create.py | 2 +- tests/unit/borg/test_repo_delete.py | 8 +- tests/unit/borg/test_repo_info.py | 50 +++--- tests/unit/borg/test_repo_list.py | 26 ++-- tests/unit/borg/test_transfer.py | 36 ++--- tests/unit/borg/test_version.py | 2 +- tests/unit/hooks/data_source/test_mariadb.py | 82 +++++----- tests/unit/hooks/data_source/test_mysql.py | 82 +++++----- .../unit/hooks/data_source/test_postgresql.py | 143 +++++++++--------- tests/unit/hooks/test_command.py | 12 +- tests/unit/test_execute.py | 70 +++------ 56 files changed, 561 insertions(+), 570 deletions(-) diff --git a/NEWS b/NEWS index b8370afa..dd0bd93f 100644 --- a/NEWS +++ b/NEWS @@ -4,6 +4,8 @@ * #795: Add credential loading from file, KeePassXC, and Docker/Podman secrets. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/ + * Send the "encryption_passphrase" option to Borg via an anonymous pipe, which is more secure than + using an environment variable. * Fix another error in the Btrfs hook when a subvolume mounted at "/" is configured in borgmatic's source directories. diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index d3e347e3..2fcca67d 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -391,7 +391,7 @@ def collect_spot_check_source_paths( paths_output = borgmatic.execute.execute_command_and_capture_output( create_flags + create_positional_arguments, capture_stderr=True, - extra_environment=borgmatic.borg.environment.make_environment(config), + environment=borgmatic.borg.environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), diff --git a/borgmatic/borg/borg.py b/borgmatic/borg/borg.py index eb52894a..a2afd430 100644 --- a/borgmatic/borg/borg.py +++ b/borgmatic/borg/borg.py @@ -61,7 +61,7 @@ def run_arbitrary_borg( tuple(shlex.quote(part) for part in full_command), output_file=DO_NOT_CAPTURE, shell=True, - extra_environment=dict( + environment=dict( (environment.make_environment(config) or {}), **{ 'BORG_REPO': repository_path, diff --git a/borgmatic/borg/break_lock.py b/borgmatic/borg/break_lock.py index 478221f0..8c5533ce 100644 --- a/borgmatic/borg/break_lock.py +++ b/borgmatic/borg/break_lock.py @@ -36,7 +36,7 @@ def break_lock( execute_command( full_command, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), diff --git a/borgmatic/borg/change_passphrase.py b/borgmatic/borg/change_passphrase.py index 79f9781d..f527e204 100644 --- a/borgmatic/borg/change_passphrase.py +++ b/borgmatic/borg/change_passphrase.py @@ -56,7 +56,7 @@ def change_passphrase( full_command, output_file=borgmatic.execute.DO_NOT_CAPTURE, output_log_level=logging.ANSWER, - extra_environment=environment.make_environment(config_without_passphrase), + environment=environment.make_environment(config_without_passphrase), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 4350b8e4..00a090a2 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -182,7 +182,7 @@ def check_archives( output_file=( DO_NOT_CAPTURE if check_arguments.repair or check_arguments.progress else None ), - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, borg_exit_codes=borg_exit_codes, diff --git a/borgmatic/borg/compact.py b/borgmatic/borg/compact.py index eb16c6fa..fd248cbf 100644 --- a/borgmatic/borg/compact.py +++ b/borgmatic/borg/compact.py @@ -49,7 +49,7 @@ def compact_segments( execute_command( full_command, output_log_level=logging.INFO, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 924c50e7..9436afd4 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -143,7 +143,7 @@ def collect_special_file_paths( + ('--dry-run', '--list'), capture_stderr=True, working_directory=working_directory, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), ) @@ -409,7 +409,7 @@ def create_archive( output_log_level, output_file, working_directory=working_directory, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), borg_local_path=local_path, borg_exit_codes=borg_exit_codes, ) @@ -417,7 +417,7 @@ def create_archive( return execute_command_and_capture_output( create_flags + create_positional_arguments, working_directory=working_directory, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), borg_local_path=local_path, borg_exit_codes=borg_exit_codes, ) @@ -427,7 +427,7 @@ def create_archive( output_log_level, output_file, working_directory=working_directory, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), borg_local_path=local_path, borg_exit_codes=borg_exit_codes, ) diff --git a/borgmatic/borg/delete.py b/borgmatic/borg/delete.py index d53a1561..d967582c 100644 --- a/borgmatic/borg/delete.py +++ b/borgmatic/borg/delete.py @@ -128,7 +128,7 @@ def delete_archives( borgmatic.execute.execute_command( command, output_log_level=logging.ANSWER, - extra_environment=borgmatic.borg.environment.make_environment(config), + environment=borgmatic.borg.environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), diff --git a/borgmatic/borg/environment.py b/borgmatic/borg/environment.py index 6e77552e..273fa91b 100644 --- a/borgmatic/borg/environment.py +++ b/borgmatic/borg/environment.py @@ -10,13 +10,10 @@ OPTION_TO_ENVIRONMENT_VARIABLE = { 'borg_files_cache_ttl': 'BORG_FILES_CACHE_TTL', 'borg_security_directory': 'BORG_SECURITY_DIR', 'borg_keys_directory': 'BORG_KEYS_DIR', - 'encryption_passphrase': 'BORG_PASSPHRASE', 'ssh_command': 'BORG_RSH', 'temporary_directory': 'TMPDIR', } -CREDENTIAL_OPTIONS = {'encryption_passphrase'} - DEFAULT_BOOL_OPTION_TO_DOWNCASE_ENVIRONMENT_VARIABLE = { 'relocated_repo_access_is_ok': 'BORG_RELOCATED_REPO_ACCESS_IS_OK', 'unknown_unencrypted_repo_access_is_ok': 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK', @@ -29,27 +26,49 @@ DEFAULT_BOOL_OPTION_TO_UPPERCASE_ENVIRONMENT_VARIABLE = { def make_environment(config): ''' - Given a borgmatic configuration dict, return its options converted to a Borg environment - variable dict. + Given a borgmatic configuration dict, convert it to a Borg environment variable dict, merge it + with a copy of the current environment variables and return the result. Do not reuse this environment across multiple Borg invocations, because it can include references to resources like anonymous pipes for passphrases—which can only be consumed once. + + Here's how native Borg precedence works for a few of the environment variables: + + 1. BORG_PASSPHRASE, if set, is used first. + 2. BORG_PASSCOMMAND is used only if BORG_PASSPHRASE isn't set. + 3. BORG_PASSPHRASE_FD is used only if neither of the above are set. + + In borgmatic, we want to simulate this precedence order, but there are some additional + complications. First, values can come from either configuration or from environment variables + set outside borgmatic and configured options should take precedence. Second, when borgmatic gets + a passphrase—directly from configuration or indirectly via a credential hook or a passcommand—we + want to pass that passphrase to Borg via an anonymous pipe (+ BORG_PASSPHRASE_FD), since that's + more secure than using an environment variable (BORG_PASSPHRASE). ''' - environment = {} + environment = dict(os.environ) for option_name, environment_variable_name in OPTION_TO_ENVIRONMENT_VARIABLE.items(): value = config.get(option_name) - if option_name in CREDENTIAL_OPTIONS and value is not None: - value = borgmatic.hooks.credential.parse.resolve_credential(value, config) - if value is not None: environment[environment_variable_name] = str(value) - passphrase = borgmatic.borg.passcommand.get_passphrase_from_passcommand(config) + if 'encryption_passphrase' in config: + environment.pop('BORG_PASSPHRASE', None) - # If the passcommand produced a passphrase, send it to Borg via an anonymous pipe. - if passphrase: + if 'encryption_passcommand' in config: + environment.pop('BORG_PASSCOMMAND', None) + + passphrase = borgmatic.hooks.credential.parse.resolve_credential( + config.get('encryption_passphrase'), config + ) + + if passphrase is None: + passphrase = borgmatic.borg.passcommand.get_passphrase_from_passcommand(config) + + # If there's a passphrase (from configuration, from a configured credential, or from a + # configured passcommand), send it to Borg via an anonymous pipe. + if passphrase is not None: read_file_descriptor, write_file_descriptor = os.pipe() os.write(write_file_descriptor, passphrase.encode('utf-8')) os.close(write_file_descriptor) diff --git a/borgmatic/borg/export_key.py b/borgmatic/borg/export_key.py index 768b52cc..0d050b16 100644 --- a/borgmatic/borg/export_key.py +++ b/borgmatic/borg/export_key.py @@ -67,7 +67,7 @@ def export_key( full_command, output_file=output_file, output_log_level=logging.ANSWER, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), diff --git a/borgmatic/borg/export_tar.py b/borgmatic/borg/export_tar.py index 728755d5..d8283535 100644 --- a/borgmatic/borg/export_tar.py +++ b/borgmatic/borg/export_tar.py @@ -70,7 +70,7 @@ def export_tar_archive( full_command, output_file=DO_NOT_CAPTURE if destination_path == '-' else None, output_log_level=output_log_level, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index d599b5b6..5097421c 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -58,7 +58,7 @@ def extract_last_archive_dry_run( execute_command( full_extract_command, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), @@ -154,7 +154,7 @@ def extract_archive( return execute_command( full_command, output_file=DO_NOT_CAPTURE, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=full_destination_path, borg_local_path=local_path, borg_exit_codes=borg_exit_codes, @@ -166,7 +166,7 @@ def extract_archive( full_command, output_file=subprocess.PIPE, run_to_completion=False, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=full_destination_path, borg_local_path=local_path, borg_exit_codes=borg_exit_codes, @@ -176,7 +176,7 @@ def extract_archive( # if the restore paths don't exist in the archive. execute_command( full_command, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=full_destination_path, borg_local_path=local_path, borg_exit_codes=borg_exit_codes, diff --git a/borgmatic/borg/info.py b/borgmatic/borg/info.py index 696048c4..bde80a97 100644 --- a/borgmatic/borg/info.py +++ b/borgmatic/borg/info.py @@ -102,7 +102,7 @@ def display_archives_info( json_info = execute_command_and_capture_output( json_command, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, borg_exit_codes=borg_exit_codes, @@ -116,7 +116,7 @@ def display_archives_info( execute_command( main_command, output_log_level=logging.ANSWER, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, borg_exit_codes=borg_exit_codes, diff --git a/borgmatic/borg/list.py b/borgmatic/borg/list.py index 755b05c9..fad36e77 100644 --- a/borgmatic/borg/list.py +++ b/borgmatic/borg/list.py @@ -124,7 +124,7 @@ def capture_archive_listing( local_path, remote_path, ), - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), @@ -221,7 +221,7 @@ def list_archive( local_path, remote_path, ), - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=borg_exit_codes, @@ -257,7 +257,7 @@ def list_archive( execute_command( main_command, output_log_level=logging.ANSWER, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=borg_exit_codes, diff --git a/borgmatic/borg/mount.py b/borgmatic/borg/mount.py index 06c6422c..627c42d3 100644 --- a/borgmatic/borg/mount.py +++ b/borgmatic/borg/mount.py @@ -66,7 +66,7 @@ def mount_archive( execute_command( full_command, output_file=DO_NOT_CAPTURE, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), @@ -75,7 +75,7 @@ def mount_archive( execute_command( full_command, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), diff --git a/borgmatic/borg/passcommand.py b/borgmatic/borg/passcommand.py index f70a1392..79132fb4 100644 --- a/borgmatic/borg/passcommand.py +++ b/borgmatic/borg/passcommand.py @@ -9,21 +9,14 @@ logger = logging.getLogger(__name__) @functools.cache -def run_passcommand(passcommand, passphrase_configured, working_directory): +def run_passcommand(passcommand, working_directory): ''' Run the given passcommand using the given working directory and return the passphrase produced - by the command. But bail first if a passphrase is already configured; this mimics Borg's - behavior. + by the command. Cache the results so that the passcommand only needs to run—and potentially prompt the user—once per borgmatic invocation. ''' - if passcommand and passphrase_configured: - logger.warning( - 'Ignoring the "encryption_passcommand" option because "encryption_passphrase" is set' - ) - return None - return borgmatic.execute.execute_command_and_capture_output( shlex.split(passcommand), working_directory=working_directory, @@ -44,7 +37,4 @@ def get_passphrase_from_passcommand(config): if not passcommand: return None - passphrase = config.get('encryption_passphrase') - working_directory = borgmatic.config.paths.get_working_directory(config) - - return run_passcommand(passcommand, bool(passphrase is not None), working_directory) + return run_passcommand(passcommand, borgmatic.config.paths.get_working_directory(config)) diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index b9765fac..fa43d5e0 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -96,7 +96,7 @@ def prune_archives( execute_command( full_command, output_log_level=output_log_level, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), diff --git a/borgmatic/borg/repo_create.py b/borgmatic/borg/repo_create.py index 7ccf85ba..b4cde394 100644 --- a/borgmatic/borg/repo_create.py +++ b/borgmatic/borg/repo_create.py @@ -98,7 +98,7 @@ def create_repository( execute_command( repo_create_command, output_file=DO_NOT_CAPTURE, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), diff --git a/borgmatic/borg/repo_delete.py b/borgmatic/borg/repo_delete.py index e2241a3e..fa66d3c0 100644 --- a/borgmatic/borg/repo_delete.py +++ b/borgmatic/borg/repo_delete.py @@ -88,7 +88,7 @@ def delete_repository( if repo_delete_arguments.force or repo_delete_arguments.cache_only else borgmatic.execute.DO_NOT_CAPTURE ), - extra_environment=borgmatic.borg.environment.make_environment(config), + environment=borgmatic.borg.environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), diff --git a/borgmatic/borg/repo_info.py b/borgmatic/borg/repo_info.py index 0c37a052..d3ea2575 100644 --- a/borgmatic/borg/repo_info.py +++ b/borgmatic/borg/repo_info.py @@ -56,7 +56,7 @@ def display_repository_info( if repo_info_arguments.json: return execute_command_and_capture_output( full_command, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, borg_exit_codes=borg_exit_codes, @@ -65,7 +65,7 @@ def display_repository_info( execute_command( full_command, output_log_level=logging.ANSWER, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, borg_exit_codes=borg_exit_codes, diff --git a/borgmatic/borg/repo_list.py b/borgmatic/borg/repo_list.py index f8e46289..9722758f 100644 --- a/borgmatic/borg/repo_list.py +++ b/borgmatic/borg/repo_list.py @@ -49,7 +49,7 @@ def resolve_archive_name( output = execute_command_and_capture_output( full_command, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), @@ -164,7 +164,7 @@ def list_repository( json_listing = execute_command_and_capture_output( json_command, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, borg_exit_codes=borg_exit_codes, @@ -178,7 +178,7 @@ def list_repository( execute_command( main_command, output_log_level=logging.ANSWER, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, borg_exit_codes=borg_exit_codes, diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index 473ae0cb..3af998a6 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -57,7 +57,7 @@ def transfer_archives( full_command, output_log_level=logging.ANSWER, output_file=DO_NOT_CAPTURE if transfer_arguments.progress else None, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), diff --git a/borgmatic/borg/version.py b/borgmatic/borg/version.py index e9cc4035..1c07dce6 100644 --- a/borgmatic/borg/version.py +++ b/borgmatic/borg/version.py @@ -21,7 +21,7 @@ def local_borg_version(config, local_path='borg'): ) output = execute_command_and_capture_output( full_command, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), diff --git a/borgmatic/execute.py b/borgmatic/execute.py index ca07458a..df58c833 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -1,7 +1,6 @@ import collections import enum import logging -import os import select import subprocess import textwrap @@ -243,6 +242,9 @@ def mask_command_secrets(full_command): MAX_LOGGED_COMMAND_LENGTH = 1000 +PREFIXES_OF_ENVIRONMENT_VARIABLES_TO_LOG = ('BORG_', 'PG', 'MARIADB_', 'MYSQL_') + + def log_command(full_command, input_file=None, output_file=None, environment=None): ''' Log the given command (a sequence of command/argument strings), along with its input/output file @@ -251,7 +253,14 @@ def log_command(full_command, input_file=None, output_file=None, environment=Non logger.debug( textwrap.shorten( ' '.join( - tuple(f'{key}=***' for key in (environment or {}).keys()) + tuple( + f'{key}=***' + for key in (environment or {}).keys() + if any( + key.startswith(prefix) + for prefix in PREFIXES_OF_ENVIRONMENT_VARIABLES_TO_LOG + ) + ) + mask_command_secrets(full_command) ), width=MAX_LOGGED_COMMAND_LENGTH, @@ -274,7 +283,7 @@ def execute_command( output_file=None, input_file=None, shell=False, - extra_environment=None, + environment=None, working_directory=None, borg_local_path=None, borg_exit_codes=None, @@ -284,18 +293,17 @@ def execute_command( Execute the given command (a sequence of command/argument strings) and log its output at the given log level. If an open output file object is given, then write stdout to the file and only log stderr. If an open input file object is given, then read stdin from the file. If shell is - True, execute the command within a shell. If an extra environment dict is given, then use it to - augment the current environment, and pass the result into the command. If a working directory is - given, use that as the present working directory when running the command. If a Borg local path - is given, and the command matches it (regardless of arguments), treat exit code 1 as a warning - instead of an error. But if Borg exit codes are given as a sequence of exit code configuration - dicts, then use that configuration to decide what's an error and what's a warning. If run to - completion is False, then return the process for the command without executing it to completion. + True, execute the command within a shell. If an environment variables dict is given, then pass + it into the command. If a working directory is given, use that as the present working directory + when running the command. If a Borg local path is given, and the command matches it (regardless + of arguments), treat exit code 1 as a warning instead of an error. But if Borg exit codes are + given as a sequence of exit code configuration dicts, then use that configuration to decide + what's an error and what's a warning. If run to completion is False, then return the process for + the command without executing it to completion. Raise subprocesses.CalledProcessError if an error occurs while running the command. ''' - log_command(full_command, input_file, output_file, extra_environment) - environment = {**os.environ, **extra_environment} if extra_environment else None + log_command(full_command, input_file, output_file, environment) do_not_capture = bool(output_file is DO_NOT_CAPTURE) command = ' '.join(full_command) if shell else full_command @@ -308,7 +316,7 @@ def execute_command( env=environment, cwd=working_directory, # Necessary for the passcommand credential hook to work. - close_fds=not bool((extra_environment or {}).get('BORG_PASSPHRASE_FD')), + close_fds=not bool((environment or {}).get('BORG_PASSPHRASE_FD')), ) if not run_to_completion: return process @@ -327,7 +335,7 @@ def execute_command_and_capture_output( full_command, capture_stderr=False, shell=False, - extra_environment=None, + environment=None, working_directory=None, borg_local_path=None, borg_exit_codes=None, @@ -335,18 +343,16 @@ def execute_command_and_capture_output( ''' Execute the given command (a sequence of command/argument strings), capturing and returning its output (stdout). If capture stderr is True, then capture and return stderr in addition to - stdout. If shell is True, execute the command within a shell. If an extra environment dict is - given, then use it to augment the current environment, and pass the result into the command. If - a working directory is given, use that as the present working directory when running the - command. If a Borg local path is given, and the command matches it (regardless of arguments), - treat exit code 1 as a warning instead of an error. But if Borg exit codes are given as a - sequence of exit code configuration dicts, then use that configuration to decide what's an error - and what's a warning. + stdout. If shell is True, execute the command within a shell. If an environment variables dict + is given, then pass it into the command. If a working directory is given, use that as the + present working directory when running the command. If a Borg local path is given, and the + command matches it (regardless of arguments), treat exit code 1 as a warning instead of an + error. But if Borg exit codes are given as a sequence of exit code configuration dicts, then use + that configuration to decide what's an error and what's a warning. Raise subprocesses.CalledProcessError if an error occurs while running the command. ''' - log_command(full_command, environment=extra_environment) - environment = {**os.environ, **extra_environment} if extra_environment else None + log_command(full_command, environment=environment) command = ' '.join(full_command) if shell else full_command try: @@ -357,7 +363,7 @@ def execute_command_and_capture_output( env=environment, cwd=working_directory, # Necessary for the passcommand credential hook to work. - close_fds=not bool((extra_environment or {}).get('BORG_PASSPHRASE_FD')), + close_fds=not bool((environment or {}).get('BORG_PASSPHRASE_FD')), ) except subprocess.CalledProcessError as error: if ( @@ -377,7 +383,7 @@ def execute_command_with_processes( output_file=None, input_file=None, shell=False, - extra_environment=None, + environment=None, working_directory=None, borg_local_path=None, borg_exit_codes=None, @@ -391,19 +397,17 @@ def execute_command_with_processes( If an open output file object is given, then write stdout to the file and only log stderr. But if output log level is None, instead suppress logging and return the captured output for (only) the given command. If an open input file object is given, then read stdin from the file. If - shell is True, execute the command within a shell. If an extra environment dict is given, then - use it to augment the current environment, and pass the result into the command. If a working - directory is given, use that as the present working directory when running the command. If a - Borg local path is given, then for any matching command or process (regardless of arguments), - treat exit code 1 as a warning instead of an error. But if Borg exit codes are given as a - sequence of exit code configuration dicts, then use that configuration to decide what's an error - and what's a warning. + shell is True, execute the command within a shell. If an environment variables dict is given, + then pass it into the command. If a working directory is given, use that as the present working + directory when running the command. If a Borg local path is given, then for any matching command + or process (regardless of arguments), treat exit code 1 as a warning instead of an error. But if + Borg exit codes are given as a sequence of exit code configuration dicts, then use that + configuration to decide what's an error and what's a warning. Raise subprocesses.CalledProcessError if an error occurs while running the command or in the upstream process. ''' - log_command(full_command, input_file, output_file, extra_environment) - environment = {**os.environ, **extra_environment} if extra_environment else None + log_command(full_command, input_file, output_file, environment) do_not_capture = bool(output_file is DO_NOT_CAPTURE) command = ' '.join(full_command) if shell else full_command @@ -419,7 +423,7 @@ def execute_command_with_processes( env=environment, cwd=working_directory, # Necessary for the passcommand credential hook to work. - close_fds=not bool((extra_environment or {}).get('BORG_PASSPHRASE_FD')), + close_fds=not bool((environment or {}).get('BORG_PASSPHRASE_FD')), ) except (subprocess.CalledProcessError, OSError): # Something has gone wrong. So vent each process' output buffer to prevent it from hanging. diff --git a/borgmatic/hooks/command.py b/borgmatic/hooks/command.py index d446ecca..9e605290 100644 --- a/borgmatic/hooks/command.py +++ b/borgmatic/hooks/command.py @@ -30,16 +30,18 @@ def interpolate_context(hook_description, command, context): def make_environment(current_environment, sys_module=sys): ''' - Given the existing system environment as a map from environment variable name to value, return - (in the same form) any extra environment variables that should be used when running command - hooks. + Given the existing system environment as a map from environment variable name to value, return a + copy of it, augmented with any extra environment variables that should be used when running + command hooks. ''' + environment = dict(current_environment) + # Detect whether we're running within a PyInstaller bundle. If so, set or clear LD_LIBRARY_PATH # based on the value of LD_LIBRARY_PATH_ORIG. This prevents library version information errors. if getattr(sys_module, 'frozen', False) and hasattr(sys_module, '_MEIPASS'): - return {'LD_LIBRARY_PATH': current_environment.get('LD_LIBRARY_PATH_ORIG', '')} + environment['LD_LIBRARY_PATH'] = environment.get('LD_LIBRARY_PATH_ORIG', '') - return {} + return environment def execute_hook(commands, umask, config_filename, description, dry_run, **context): @@ -85,7 +87,7 @@ def execute_hook(commands, umask, config_filename, description, dry_run, **conte [command], output_log_level=(logging.ERROR if description == 'on-error' else logging.WARNING), shell=True, - extra_environment=make_environment(os.environ), + environment=make_environment(os.environ), ) finally: if original_umask: diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index 47d1dfea..727932ee 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -26,7 +26,7 @@ def make_dump_path(base_directory): # pragma: no cover SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys') -def database_names_to_dump(database, config, extra_environment, dry_run): +def database_names_to_dump(database, config, environment, dry_run): ''' Given a requested database config and a configuration dict, return the corresponding sequence of database names to dump. In the case of "all", query for the names of databases on the configured @@ -58,9 +58,7 @@ def database_names_to_dump(database, config, extra_environment, dry_run): + ('--execute', 'show schemas') ) logger.debug('Querying for "all" MariaDB databases to dump') - show_output = execute_command_and_capture_output( - show_command, extra_environment=extra_environment - ) + show_output = execute_command_and_capture_output(show_command, environment=environment) return tuple( show_name @@ -70,7 +68,7 @@ def database_names_to_dump(database, config, extra_environment, dry_run): def execute_dump_command( - database, config, dump_path, database_names, extra_environment, dry_run, dry_run_label + database, config, dump_path, database_names, environment, dry_run, dry_run_label ): ''' Kick off a dump for the given MariaDB database (provided as a configuration dict) to a named @@ -125,7 +123,7 @@ def execute_dump_command( return execute_command( dump_command, - extra_environment=extra_environment, + environment=environment, run_to_completion=False, ) @@ -167,16 +165,19 @@ def dump_data_sources( for database in databases: dump_path = make_dump_path(borgmatic_runtime_directory) - extra_environment = ( - { - 'MYSQL_PWD': borgmatic.hooks.credential.parse.resolve_credential( - database['password'], config - ) - } - if 'password' in database - else None + environment = dict( + os.environ, + **( + { + 'MYSQL_PWD': borgmatic.hooks.credential.parse.resolve_credential( + database['password'], config + ) + } + if 'password' in database + else {} + ), ) - dump_database_names = database_names_to_dump(database, config, extra_environment, dry_run) + dump_database_names = database_names_to_dump(database, config, environment, dry_run) if not dump_database_names: if dry_run: @@ -194,7 +195,7 @@ def dump_data_sources( config, dump_path, (dump_name,), - extra_environment, + environment, dry_run, dry_run_label, ) @@ -206,7 +207,7 @@ def dump_data_sources( config, dump_path, dump_database_names, - extra_environment, + environment, dry_run, dry_run_label, ) @@ -307,7 +308,7 @@ def restore_data_source_dump( + (('--protocol', 'tcp') if hostname or port else ()) + (('--user', username) if username else ()) ) - extra_environment = {'MYSQL_PWD': password} if password else None + environment = dict(os.environ, **({'MYSQL_PWD': password} if password else {})) logger.debug(f"Restoring MariaDB database {data_source['name']}{dry_run_label}") if dry_run: @@ -320,5 +321,5 @@ def restore_data_source_dump( [extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment=extra_environment, + environment=environment, ) diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index aa5a954e..ffa2bc3f 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -26,7 +26,7 @@ def make_dump_path(base_directory): # pragma: no cover SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys') -def database_names_to_dump(database, config, extra_environment, dry_run): +def database_names_to_dump(database, config, environment, dry_run): ''' Given a requested database config and a configuration dict, return the corresponding sequence of database names to dump. In the case of "all", query for the names of databases on the configured @@ -58,9 +58,7 @@ def database_names_to_dump(database, config, extra_environment, dry_run): + ('--execute', 'show schemas') ) logger.debug('Querying for "all" MySQL databases to dump') - show_output = execute_command_and_capture_output( - show_command, extra_environment=extra_environment - ) + show_output = execute_command_and_capture_output(show_command, environment=environment) return tuple( show_name @@ -70,7 +68,7 @@ def database_names_to_dump(database, config, extra_environment, dry_run): def execute_dump_command( - database, config, dump_path, database_names, extra_environment, dry_run, dry_run_label + database, config, dump_path, database_names, environment, dry_run, dry_run_label ): ''' Kick off a dump for the given MySQL/MariaDB database (provided as a configuration dict) to a @@ -124,7 +122,7 @@ def execute_dump_command( return execute_command( dump_command, - extra_environment=extra_environment, + environment=environment, run_to_completion=False, ) @@ -166,16 +164,19 @@ def dump_data_sources( for database in databases: dump_path = make_dump_path(borgmatic_runtime_directory) - extra_environment = ( - { - 'MYSQL_PWD': borgmatic.hooks.credential.parse.resolve_credential( - database['password'], config - ) - } - if 'password' in database - else None + environment = dict( + os.environ, + **( + { + 'MYSQL_PWD': borgmatic.hooks.credential.parse.resolve_credential( + database['password'], config + ) + } + if 'password' in database + else {} + ), ) - dump_database_names = database_names_to_dump(database, config, extra_environment, dry_run) + dump_database_names = database_names_to_dump(database, config, environment, dry_run) if not dump_database_names: if dry_run: @@ -193,7 +194,7 @@ def dump_data_sources( config, dump_path, (dump_name,), - extra_environment, + environment, dry_run, dry_run_label, ) @@ -205,7 +206,7 @@ def dump_data_sources( config, dump_path, dump_database_names, - extra_environment, + environment, dry_run, dry_run_label, ) @@ -306,7 +307,7 @@ def restore_data_source_dump( + (('--protocol', 'tcp') if hostname or port else ()) + (('--user', username) if username else ()) ) - extra_environment = {'MYSQL_PWD': password} if password else None + environment = dict(os.environ, **({'MYSQL_PWD': password} if password else {})) logger.debug(f"Restoring MySQL database {data_source['name']}{dry_run_label}") if dry_run: @@ -319,5 +320,5 @@ def restore_data_source_dump( [extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment=extra_environment, + environment=environment, ) diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index a9eafea0..8cf17518 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -25,16 +25,16 @@ def make_dump_path(base_directory): # pragma: no cover return dump.make_data_source_dump_path(base_directory, 'postgresql_databases') -def make_extra_environment(database, config, restore_connection_params=None): +def make_environment(database, config, restore_connection_params=None): ''' - Make the extra_environment dict from the given database configuration. If restore connection - params are given, this is for a restore operation. + Make an environment dict from the current environment variables and the given database + configuration. If restore connection params are given, this is for a restore operation. ''' - extra = dict() + environment = dict(os.environ) try: if restore_connection_params: - extra['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential( + environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential( ( restore_connection_params.get('password') or database.get('restore_password', database['password']) @@ -42,30 +42,30 @@ def make_extra_environment(database, config, restore_connection_params=None): config, ) else: - extra['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential( + environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential( database['password'], config ) except (AttributeError, KeyError): pass if 'ssl_mode' in database: - extra['PGSSLMODE'] = database['ssl_mode'] + environment['PGSSLMODE'] = database['ssl_mode'] if 'ssl_cert' in database: - extra['PGSSLCERT'] = database['ssl_cert'] + environment['PGSSLCERT'] = database['ssl_cert'] if 'ssl_key' in database: - extra['PGSSLKEY'] = database['ssl_key'] + environment['PGSSLKEY'] = database['ssl_key'] if 'ssl_root_cert' in database: - extra['PGSSLROOTCERT'] = database['ssl_root_cert'] + environment['PGSSLROOTCERT'] = database['ssl_root_cert'] if 'ssl_crl' in database: - extra['PGSSLCRL'] = database['ssl_crl'] + environment['PGSSLCRL'] = database['ssl_crl'] - return extra + return environment EXCLUDED_DATABASE_NAMES = ('template0', 'template1') -def database_names_to_dump(database, config, extra_environment, dry_run): +def database_names_to_dump(database, config, environment, dry_run): ''' Given a requested database config and a configuration dict, return the corresponding sequence of database names to dump. In the case of "all" when a database format is given, query for the @@ -100,9 +100,7 @@ def database_names_to_dump(database, config, extra_environment, dry_run): + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ()) ) logger.debug('Querying for "all" PostgreSQL databases to dump') - list_output = execute_command_and_capture_output( - list_command, extra_environment=extra_environment - ) + list_output = execute_command_and_capture_output(list_command, environment=environment) return tuple( row[0] @@ -149,9 +147,9 @@ def dump_data_sources( logger.info(f'Dumping PostgreSQL databases{dry_run_label}') for database in databases: - extra_environment = make_extra_environment(database, config) + environment = make_environment(database, config) dump_path = make_dump_path(borgmatic_runtime_directory) - dump_database_names = database_names_to_dump(database, config, extra_environment, dry_run) + dump_database_names = database_names_to_dump(database, config, environment, dry_run) if not dump_database_names: if dry_run: @@ -225,7 +223,7 @@ def dump_data_sources( execute_command( command, shell=True, - extra_environment=extra_environment, + environment=environment, ) else: dump.create_named_pipe_for_dump(dump_filename) @@ -233,7 +231,7 @@ def dump_data_sources( execute_command( command, shell=True, - extra_environment=extra_environment, + environment=environment, run_to_completion=False, ) ) @@ -369,9 +367,7 @@ def restore_data_source_dump( ) ) - extra_environment = make_extra_environment( - data_source, config, restore_connection_params=connection_params - ) + environment = make_environment(data_source, config, restore_connection_params=connection_params) logger.debug(f"Restoring PostgreSQL database {data_source['name']}{dry_run_label}") if dry_run: @@ -384,6 +380,6 @@ def restore_data_source_dump( [extract_process] if extract_process else [], output_log_level=logging.DEBUG, input_file=extract_process.stdout if extract_process else None, - extra_environment=extra_environment, + environment=environment, ) - execute_command(analyze_command, extra_environment=extra_environment) + execute_command(analyze_command, environment=environment) diff --git a/tests/unit/borg/test_borg.py b/tests/unit/borg/test_borg.py index 573ee21e..5f346917 100644 --- a/tests/unit/borg/test_borg.py +++ b/tests/unit/borg/test_borg.py @@ -17,7 +17,7 @@ def test_run_arbitrary_borg_calls_borg_with_flags(): ('borg', 'break-lock', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -41,7 +41,7 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_flag(): ('borg', 'break-lock', '--info', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -66,7 +66,7 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_flag(): ('borg', 'break-lock', '--debug', '--show-rc', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -94,7 +94,7 @@ def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_flags(): ('borg', 'break-lock', '--lock-wait', '5', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -118,7 +118,7 @@ def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_flag(): ('borg', 'break-lock', "'::$ARCHIVE'"), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': 'archive'}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': 'archive'}, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -143,7 +143,7 @@ def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path(): ('borg1', 'break-lock', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg1', borg_exit_codes=None, @@ -169,7 +169,7 @@ def test_run_arbitrary_borg_with_exit_codes_calls_borg_using_them(): ('borg', 'break-lock', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg', borg_exit_codes=borg_exit_codes, @@ -195,7 +195,7 @@ def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_flags() ('borg', 'break-lock', '--remote-path', 'borg1', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -222,7 +222,7 @@ def test_run_arbitrary_borg_with_remote_path_injection_attack_gets_escaped(): ('borg', 'break-lock', '--remote-path', "'borg1; naughty-command'", '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -247,7 +247,7 @@ def test_run_arbitrary_borg_passes_borg_specific_flags_to_borg(): ('borg', 'list', '--progress', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -271,7 +271,7 @@ def test_run_arbitrary_borg_omits_dash_dash_in_flags_passed_to_borg(): ('borg', 'break-lock', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -295,7 +295,7 @@ def test_run_arbitrary_borg_without_borg_specific_flags_does_not_raise(): ('borg',), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -319,7 +319,7 @@ def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_injected_flags ('borg', 'key', 'export', '--info', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -344,7 +344,7 @@ def test_run_arbitrary_borg_passes_debug_sub_command_to_borg_before_injected_fla ('borg', 'debug', 'dump-manifest', '--info', '::', 'path'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -371,7 +371,7 @@ def test_run_arbitrary_borg_calls_borg_with_working_directory(): ('borg', 'break-lock', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory='/working/dir', borg_local_path='borg', borg_exit_codes=None, diff --git a/tests/unit/borg/test_break_lock.py b/tests/unit/borg/test_break_lock.py index 202cd7d6..8368a19a 100644 --- a/tests/unit/borg/test_break_lock.py +++ b/tests/unit/borg/test_break_lock.py @@ -14,7 +14,7 @@ def insert_execute_command_mock(command, working_directory=None, borg_exit_codes ) flexmock(module).should_receive('execute_command').with_args( command, - extra_environment=None, + environment=None, working_directory=working_directory, borg_local_path=command[0], borg_exit_codes=borg_exit_codes, diff --git a/tests/unit/borg/test_change_passphrase.py b/tests/unit/borg/test_change_passphrase.py index c7b75c7c..ef4b7b35 100644 --- a/tests/unit/borg/test_change_passphrase.py +++ b/tests/unit/borg/test_change_passphrase.py @@ -25,7 +25,7 @@ def insert_execute_command_mock( command, output_file=output_file, output_log_level=module.logging.ANSWER, - extra_environment=None, + environment=None, working_directory=working_directory, borg_local_path=command[0], borg_exit_codes=borg_exit_codes, diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index 3fadc2fd..cb3ce67c 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -18,7 +18,7 @@ def insert_execute_command_mock( flexmock(module).should_receive('execute_command').with_args( command, output_file=output_file, - extra_environment=None, + environment=None, working_directory=working_directory, borg_local_path=command[0], borg_exit_codes=borg_exit_codes, @@ -342,7 +342,7 @@ def test_check_archives_with_progress_passes_through_to_borg(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'check', '--progress', 'repo'), output_file=module.DO_NOT_CAPTURE, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -377,7 +377,7 @@ def test_check_archives_with_repair_passes_through_to_borg(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'check', '--repair', 'repo'), output_file=module.DO_NOT_CAPTURE, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -412,7 +412,7 @@ def test_check_archives_with_max_duration_flag_passes_through_to_borg(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'check', '--max-duration', '33', 'repo'), output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -447,7 +447,7 @@ def test_check_archives_with_max_duration_option_passes_through_to_borg(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'check', '--max-duration', '33', 'repo'), output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -610,7 +610,7 @@ def test_check_archives_with_max_duration_flag_overrides_max_duration_option(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'check', '--max-duration', '44', 'repo'), output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -963,7 +963,7 @@ def test_check_archives_with_match_archives_passes_through_to_borg(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'check', '--match-archives', 'foo-*', 'repo'), output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, diff --git a/tests/unit/borg/test_compact.py b/tests/unit/borg/test_compact.py index 18df3d13..882dac69 100644 --- a/tests/unit/borg/test_compact.py +++ b/tests/unit/borg/test_compact.py @@ -17,7 +17,7 @@ def insert_execute_command_mock( flexmock(module).should_receive('execute_command').with_args( compact_command, output_log_level=output_log_level, - extra_environment=None, + environment=None, working_directory=working_directory, borg_local_path=compact_command[0], borg_exit_codes=borg_exit_codes, diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 6015004a..94e02425 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -296,7 +296,7 @@ def test_collect_special_file_paths_omits_exclude_no_dump_flag_from_command(): ('borg', 'create', '--dry-run', '--list'), capture_stderr=True, working_directory=None, - extra_environment=None, + environment=None, borg_local_path='borg', borg_exit_codes=None, ).and_return('Processing files ...\n- /foo\n+ /bar\n- /baz').once() @@ -969,7 +969,7 @@ def test_create_archive_calls_borg_with_parameters(): borg_local_path='borg', borg_exit_codes=None, working_directory=None, - extra_environment=None, + environment=None, ) module.create_archive( @@ -1003,7 +1003,7 @@ def test_create_archive_calls_borg_with_environment(): borg_local_path='borg', borg_exit_codes=None, working_directory=None, - extra_environment=environment, + environment=environment, ) module.create_archive( @@ -1036,7 +1036,7 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter(): borg_local_path='borg', borg_exit_codes=None, working_directory=None, - extra_environment=None, + environment=None, ) insert_logging_mock(logging.INFO) @@ -1066,7 +1066,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output(): flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'create', '--json') + REPO_ARCHIVE, working_directory=None, - extra_environment=None, + environment=None, borg_local_path='borg', borg_exit_codes=None, ) @@ -1103,7 +1103,7 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter(): borg_local_path='borg', borg_exit_codes=None, working_directory=None, - extra_environment=None, + environment=None, ) insert_logging_mock(logging.DEBUG) @@ -1133,7 +1133,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output(): flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'create', '--json') + REPO_ARCHIVE, working_directory=None, - extra_environment=None, + environment=None, borg_local_path='borg', borg_exit_codes=None, ) @@ -1172,7 +1172,7 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats(): borg_local_path='borg', borg_exit_codes=None, working_directory=None, - extra_environment=None, + environment=None, ) insert_logging_mock(logging.INFO) @@ -1209,7 +1209,7 @@ def test_create_archive_with_working_directory_calls_borg_with_working_directory borg_local_path='borg', borg_exit_codes=None, working_directory='/working/dir', - extra_environment=None, + environment=None, ) module.create_archive( @@ -1244,7 +1244,7 @@ def test_create_archive_with_exit_codes_calls_borg_using_them(): borg_local_path='borg', borg_exit_codes=borg_exit_codes, working_directory=None, - extra_environment=None, + environment=None, ) module.create_archive( @@ -1278,7 +1278,7 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_ou borg_local_path='borg', borg_exit_codes=None, working_directory=None, - extra_environment=None, + environment=None, ) module.create_archive( @@ -1316,7 +1316,7 @@ def test_create_archive_with_files_calls_borg_with_answer_output_log_level(): borg_local_path='borg', borg_exit_codes=None, working_directory=None, - extra_environment=None, + environment=None, ) module.create_archive( @@ -1350,7 +1350,7 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para borg_local_path='borg', borg_exit_codes=None, working_directory=None, - extra_environment=None, + environment=None, ) insert_logging_mock(logging.INFO) @@ -1385,7 +1385,7 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter(): borg_local_path='borg', borg_exit_codes=None, working_directory=None, - extra_environment=None, + environment=None, ) module.create_archive( @@ -1431,7 +1431,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr borg_local_path='borg', borg_exit_codes=None, working_directory=None, - extra_environment=None, + environment=None, ) flexmock(module).should_receive('execute_command_with_processes').with_args( create_command, @@ -1441,7 +1441,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr borg_local_path='borg', borg_exit_codes=None, working_directory=None, - extra_environment=None, + environment=None, ) module.create_archive( @@ -1472,7 +1472,7 @@ def test_create_archive_with_json_calls_borg_with_json_flag(): flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'create', '--json') + REPO_ARCHIVE, working_directory=None, - extra_environment=None, + environment=None, borg_local_path='borg', borg_exit_codes=None, ).and_return('[]') @@ -1506,7 +1506,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_flag(): flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'create', '--json') + REPO_ARCHIVE, working_directory=None, - extra_environment=None, + environment=None, borg_local_path='borg', borg_exit_codes=None, ).and_return('[]') @@ -1547,7 +1547,7 @@ def test_create_archive_calls_borg_with_working_directory(): borg_local_path='borg', borg_exit_codes=None, working_directory='/working/dir', - extra_environment=None, + environment=None, ) module.create_archive( diff --git a/tests/unit/borg/test_delete.py b/tests/unit/borg/test_delete.py index c3bf9db7..6d8614b6 100644 --- a/tests/unit/borg/test_delete.py +++ b/tests/unit/borg/test_delete.py @@ -370,9 +370,9 @@ def test_delete_archives_calls_borg_delete_with_working_directory(): flexmock(module.borgmatic.borg.repo_delete).should_receive('delete_repository').never() command = flexmock() flexmock(module).should_receive('make_delete_command').and_return(command) - extra_environment = flexmock() + environment = flexmock() flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( - extra_environment + environment ) flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return( '/working/dir' @@ -380,7 +380,7 @@ def test_delete_archives_calls_borg_delete_with_working_directory(): flexmock(module.borgmatic.execute).should_receive('execute_command').with_args( command, output_log_level=logging.ANSWER, - extra_environment=extra_environment, + environment=environment, working_directory='/working/dir', borg_local_path='borg', borg_exit_codes=None, diff --git a/tests/unit/borg/test_environment.py b/tests/unit/borg/test_environment.py index a63c3470..28ce3446 100644 --- a/tests/unit/borg/test_environment.py +++ b/tests/unit/borg/test_environment.py @@ -4,6 +4,7 @@ from borgmatic.borg import environment as module def test_make_environment_with_passcommand_should_call_it_and_set_passphrase_file_descriptor_in_environment(): + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.borg.passcommand).should_receive( 'get_passphrase_from_passcommand' ).and_return('passphrase') @@ -14,133 +15,148 @@ def test_make_environment_with_passcommand_should_call_it_and_set_passphrase_fil environment = module.make_environment({'encryption_passcommand': 'command'}) - assert not environment.get('BORG_PASSCOMMAND') + assert environment.get('BORG_PASSCOMMAND') is None assert environment.get('BORG_PASSPHRASE_FD') == '3' -def test_make_environment_with_passphrase_should_set_environment(): +def test_make_environment_with_passphrase_should_set_passphrase_file_descriptor_in_environment(): + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.borg.passcommand).should_receive( 'get_passphrase_from_passcommand' ).and_return(None) - flexmock(module.os).should_receive('pipe').never() - flexmock(module.os.environ).should_receive('get').and_return(None) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('pipe').and_return((3, 4)) + flexmock(module.os).should_receive('write') + flexmock(module.os).should_receive('close') + flexmock(module.os).should_receive('set_inheritable') environment = module.make_environment({'encryption_passphrase': 'pass'}) - assert environment.get('BORG_PASSPHRASE') == 'pass' + assert environment.get('BORG_PASSPHRASE') is None + assert environment.get('BORG_PASSPHRASE_FD') == '3' -def test_make_environment_with_credential_tag_passphrase_should_load_it_and_set_environment(): +def test_make_environment_with_credential_tag_passphrase_should_load_it_and_set_passphrase_file_descriptor_in_environment(): + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) config = {'encryption_passphrase': '{credential systemd pass}'} flexmock(module.borgmatic.borg.passcommand).should_receive( 'get_passphrase_from_passcommand' ).and_return(None) - flexmock(module.os).should_receive('pipe').never() - flexmock(module.os.environ).should_receive('get').and_return(None) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential', ).with_args('{credential systemd pass}', config).and_return('pass') + flexmock(module.os).should_receive('pipe').and_return((3, 4)) + flexmock(module.os).should_receive('write') + flexmock(module.os).should_receive('close') + flexmock(module.os).should_receive('set_inheritable') environment = module.make_environment(config) - assert environment.get('BORG_PASSPHRASE') == 'pass' + assert environment.get('BORG_PASSPHRASE') is None + assert environment.get('BORG_PASSPHRASE_FD') == '3' def test_make_environment_with_ssh_command_should_set_environment(): + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.borg.passcommand).should_receive( 'get_passphrase_from_passcommand' ).and_return(None) flexmock(module.os).should_receive('pipe').never() - flexmock(module.os.environ).should_receive('get').and_return(None) environment = module.make_environment({'ssh_command': 'ssh -C'}) assert environment.get('BORG_RSH') == 'ssh -C' def test_make_environment_without_configuration_sets_certain_environment_variables(): + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.borg.passcommand).should_receive( 'get_passphrase_from_passcommand' ).and_return(None) flexmock(module.os).should_receive('pipe').never() - flexmock(module.os.environ).should_receive('get').and_return(None) environment = module.make_environment({}) # Default environment variables. assert environment == { + 'USER': 'root', 'BORG_EXIT_CODES': 'modern', 'BORG_RELOCATED_REPO_ACCESS_IS_OK': 'no', 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK': 'no', } -def test_make_environment_without_configuration_does_not_set_certain_environment_variables_if_already_set(): +def test_make_environment_without_configuration_passes_through_default_environment_variables_untouched(): + flexmock(module.os).should_receive('environ').and_return( + { + 'USER': 'root', + 'BORG_RELOCATED_REPO_ACCESS_IS_OK': 'yup', + 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK': 'nah', + } + ) flexmock(module.borgmatic.borg.passcommand).should_receive( 'get_passphrase_from_passcommand' ).and_return(None) flexmock(module.os).should_receive('pipe').never() - flexmock(module.os.environ).should_receive('get').with_args( - 'BORG_RELOCATED_REPO_ACCESS_IS_OK' - ).and_return('yup') - flexmock(module.os.environ).should_receive('get').with_args( - 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK' - ).and_return('nah') environment = module.make_environment({}) - assert environment == {'BORG_EXIT_CODES': 'modern'} + assert environment == { + 'USER': 'root', + 'BORG_RELOCATED_REPO_ACCESS_IS_OK': 'yup', + 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK': 'nah', + 'BORG_EXIT_CODES': 'modern', + } def test_make_environment_with_relocated_repo_access_true_should_set_environment_yes(): + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.borg.passcommand).should_receive( 'get_passphrase_from_passcommand' ).and_return(None) flexmock(module.os).should_receive('pipe').never() - flexmock(module.os.environ).should_receive('get').and_return(None) environment = module.make_environment({'relocated_repo_access_is_ok': True}) assert environment.get('BORG_RELOCATED_REPO_ACCESS_IS_OK') == 'yes' def test_make_environment_with_relocated_repo_access_false_should_set_environment_no(): + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.borg.passcommand).should_receive( 'get_passphrase_from_passcommand' ).and_return(None) flexmock(module.os).should_receive('pipe').never() - flexmock(module.os.environ).should_receive('get').and_return(None) environment = module.make_environment({'relocated_repo_access_is_ok': False}) assert environment.get('BORG_RELOCATED_REPO_ACCESS_IS_OK') == 'no' def test_make_environment_check_i_know_what_i_am_doing_true_should_set_environment_YES(): + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.borg.passcommand).should_receive( 'get_passphrase_from_passcommand' ).and_return(None) flexmock(module.os).should_receive('pipe').never() - flexmock(module.os.environ).should_receive('get').and_return(None) environment = module.make_environment({'check_i_know_what_i_am_doing': True}) assert environment.get('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING') == 'YES' def test_make_environment_check_i_know_what_i_am_doing_false_should_set_environment_NO(): + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.borg.passcommand).should_receive( 'get_passphrase_from_passcommand' ).and_return(None) flexmock(module.os).should_receive('pipe').never() - flexmock(module.os.environ).should_receive('get').and_return(None) environment = module.make_environment({'check_i_know_what_i_am_doing': False}) assert environment.get('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING') == 'NO' def test_make_environment_with_integer_variable_value(): + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.borg.passcommand).should_receive( 'get_passphrase_from_passcommand' ).and_return(None) flexmock(module.os).should_receive('pipe').never() - flexmock(module.os.environ).should_receive('get').and_return(None) environment = module.make_environment({'borg_files_cache_ttl': 40}) assert environment.get('BORG_FILES_CACHE_TTL') == '40' diff --git a/tests/unit/borg/test_export_key.py b/tests/unit/borg/test_export_key.py index 2d7836a3..2fac04c2 100644 --- a/tests/unit/borg/test_export_key.py +++ b/tests/unit/borg/test_export_key.py @@ -22,7 +22,7 @@ def insert_execute_command_mock( command, output_file=output_file, output_log_level=module.logging.ANSWER, - extra_environment=None, + environment=None, working_directory=working_directory, borg_local_path=command[0], borg_exit_codes=borg_exit_codes, diff --git a/tests/unit/borg/test_export_tar.py b/tests/unit/borg/test_export_tar.py index 53307127..f45026f3 100644 --- a/tests/unit/borg/test_export_tar.py +++ b/tests/unit/borg/test_export_tar.py @@ -23,7 +23,7 @@ def insert_execute_command_mock( command, output_file=None if capture else module.DO_NOT_CAPTURE, output_log_level=output_log_level, - extra_environment=None, + environment=None, working_directory=working_directory, borg_local_path=borg_local_path, borg_exit_codes=borg_exit_codes, diff --git a/tests/unit/borg/test_extract.py b/tests/unit/borg/test_extract.py index bddb2530..db244e54 100644 --- a/tests/unit/borg/test_extract.py +++ b/tests/unit/borg/test_extract.py @@ -12,7 +12,7 @@ def insert_execute_command_mock(command, destination_path=None, borg_exit_codes= flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( command, - extra_environment=None, + environment=None, working_directory=destination_path, borg_local_path=command[0], borg_exit_codes=borg_exit_codes, @@ -587,7 +587,7 @@ def test_extract_archive_calls_borg_with_progress_parameter(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'extract', '--progress', 'repo::archive'), output_file=module.DO_NOT_CAPTURE, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -639,7 +639,7 @@ def test_extract_archive_calls_borg_with_stdout_parameter_and_returns_process(): ('borg', 'extract', '--stdout', 'repo::archive'), output_file=module.subprocess.PIPE, run_to_completion=False, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -674,7 +674,7 @@ def test_extract_archive_skips_abspath_for_remote_repository(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command').with_args( ('borg', 'extract', 'server:repo::archive'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, diff --git a/tests/unit/borg/test_info.py b/tests/unit/borg/test_info.py index e66f4911..5866b4ab 100644 --- a/tests/unit/borg/test_info.py +++ b/tests/unit/borg/test_info.py @@ -514,7 +514,7 @@ def test_display_archives_info_calls_borg_with_working_directory(): ) flexmock(module).should_receive('execute_command_and_capture_output').with_args( full_command=object, - extra_environment=object, + environment=object, working_directory='/working/dir', borg_local_path=object, borg_exit_codes=object, @@ -523,7 +523,7 @@ def test_display_archives_info_calls_borg_with_working_directory(): flexmock(module).should_receive('execute_command').with_args( full_command=object, output_log_level=object, - extra_environment=object, + environment=object, working_directory='/working/dir', borg_local_path=object, borg_exit_codes=object, diff --git a/tests/unit/borg/test_list.py b/tests/unit/borg/test_list.py index 26803c70..2268e8e5 100644 --- a/tests/unit/borg/test_list.py +++ b/tests/unit/borg/test_list.py @@ -353,7 +353,7 @@ def test_list_archive_calls_borg_with_flags(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'list', 'repo::archive'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -419,7 +419,7 @@ def test_list_archive_calls_borg_with_local_path(): flexmock(module).should_receive('execute_command').with_args( ('borg2', 'list', 'repo::archive'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg2', borg_exit_codes=None, @@ -469,7 +469,7 @@ def test_list_archive_calls_borg_using_exit_codes(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'list', 'repo::archive'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=borg_exit_codes, @@ -507,7 +507,7 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -520,7 +520,7 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'list', 'repo::archive1') + glob_paths, output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -528,7 +528,7 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'list', 'repo::archive2') + glob_paths, output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -576,7 +576,7 @@ def test_list_archive_calls_borg_with_archive(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'list', 'repo::archive'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -699,7 +699,7 @@ def test_list_archive_with_archive_ignores_archive_filter_flag( flexmock(module).should_receive('execute_command').with_args( ('borg', 'list', 'repo::archive'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -759,7 +759,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-list', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -808,7 +808,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes flexmock(module).should_receive('execute_command').with_args( ('borg', 'list', '--repo', 'repo', 'archive1') + glob_paths, output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -816,7 +816,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes flexmock(module).should_receive('execute_command').with_args( ('borg', 'list', '--repo', 'repo', 'archive2') + glob_paths, output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -875,7 +875,7 @@ def test_list_archive_calls_borg_with_working_directory(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'list', 'repo::archive'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory='/working/dir', borg_local_path='borg', borg_exit_codes=None, diff --git a/tests/unit/borg/test_mount.py b/tests/unit/borg/test_mount.py index 46a92408..ee11f266 100644 --- a/tests/unit/borg/test_mount.py +++ b/tests/unit/borg/test_mount.py @@ -14,7 +14,7 @@ def insert_execute_command_mock(command, working_directory=None, borg_exit_codes ) flexmock(module).should_receive('execute_command').with_args( command, - extra_environment=None, + environment=None, working_directory=working_directory, borg_local_path=command[0], borg_exit_codes=borg_exit_codes, @@ -262,7 +262,7 @@ def test_mount_archive_calls_borg_with_foreground_parameter(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'mount', '--foreground', 'repo::archive', '/mnt'), output_file=module.DO_NOT_CAPTURE, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -337,7 +337,7 @@ def test_mount_archive_with_date_based_matching_calls_borg_with_date_based_flags 'repo', '/mnt', ), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, diff --git a/tests/unit/borg/test_passcommand.py b/tests/unit/borg/test_passcommand.py index 1a9a0e97..a6caea2e 100644 --- a/tests/unit/borg/test_passcommand.py +++ b/tests/unit/borg/test_passcommand.py @@ -3,26 +3,13 @@ from flexmock import flexmock from borgmatic.borg import passcommand as module -def test_run_passcommand_with_passphrase_configured_bails(): - module.run_passcommand.cache_clear() - flexmock(module.borgmatic.execute).should_receive('execute_command_and_capture_output').never() - - assert ( - module.run_passcommand('passcommand', passphrase_configured=True, working_directory=None) - is None - ) - - -def test_run_passcommand_without_passphrase_configured_executes_passcommand(): +def test_run_passcommand_does_not_raise(): module.run_passcommand.cache_clear() flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' - ).and_return('passphrase').once() + ).and_return('passphrase') - assert ( - module.run_passcommand('passcommand', passphrase_configured=False, working_directory=None) - == 'passphrase' - ) + assert module.run_passcommand('passcommand', working_directory=None) == 'passphrase' def test_get_passphrase_from_passcommand_with_configured_passcommand_runs_it(): @@ -30,9 +17,9 @@ def test_get_passphrase_from_passcommand_with_configured_passcommand_runs_it(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return( '/working' ) - flexmock(module).should_receive('run_passcommand').with_args( - 'command', False, '/working' - ).and_return('passphrase').once() + flexmock(module).should_receive('run_passcommand').with_args('command', '/working').and_return( + 'passphrase' + ).once() assert ( module.get_passphrase_from_passcommand( @@ -42,38 +29,10 @@ def test_get_passphrase_from_passcommand_with_configured_passcommand_runs_it(): ) -def test_get_passphrase_from_passcommand_with_configured_passphrase_and_passcommand_detects_passphrase(): - module.run_passcommand.cache_clear() - flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return( - '/working' - ) - flexmock(module).should_receive('run_passcommand').with_args( - 'command', True, '/working' - ).and_return(None).once() +def test_get_passphrase_from_passcommand_without_configured_passcommand_bails(): + flexmock(module).should_receive('run_passcommand').never() - assert ( - module.get_passphrase_from_passcommand( - {'encryption_passphrase': 'passphrase', 'encryption_passcommand': 'command'}, - ) - is None - ) - - -def test_get_passphrase_from_passcommand_with_configured_blank_passphrase_and_passcommand_detects_passphrase(): - module.run_passcommand.cache_clear() - flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return( - '/working' - ) - flexmock(module).should_receive('run_passcommand').with_args( - 'command', True, '/working' - ).and_return(None).once() - - assert ( - module.get_passphrase_from_passcommand( - {'encryption_passphrase': '', 'encryption_passcommand': 'command'}, - ) - is None - ) + assert module.get_passphrase_from_passcommand({}) is None def test_run_passcommand_caches_passcommand_after_first_call(): @@ -82,11 +41,5 @@ def test_run_passcommand_caches_passcommand_after_first_call(): 'execute_command_and_capture_output' ).and_return('passphrase').once() - assert ( - module.run_passcommand('passcommand', passphrase_configured=False, working_directory=None) - == 'passphrase' - ) - assert ( - module.run_passcommand('passcommand', passphrase_configured=False, working_directory=None) - == 'passphrase' - ) + assert module.run_passcommand('passcommand', working_directory=None) == 'passphrase' + assert module.run_passcommand('passcommand', working_directory=None) == 'passphrase' diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index 4880f391..556d695f 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -17,7 +17,7 @@ def insert_execute_command_mock( flexmock(module).should_receive('execute_command').with_args( prune_command, output_log_level=output_log_level, - extra_environment=None, + environment=None, working_directory=working_directory, borg_local_path=prune_command[0], borg_exit_codes=borg_exit_codes, @@ -497,7 +497,7 @@ def test_prune_archives_with_date_based_matching_calls_borg_with_date_based_flag 'repo', ), output_log_level=logging.INFO, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, diff --git a/tests/unit/borg/test_repo_create.py b/tests/unit/borg/test_repo_create.py index 388ebc7b..dc645f12 100644 --- a/tests/unit/borg/test_repo_create.py +++ b/tests/unit/borg/test_repo_create.py @@ -36,7 +36,7 @@ def insert_repo_create_command_mock( flexmock(module).should_receive('execute_command').with_args( repo_create_command, output_file=module.DO_NOT_CAPTURE, - extra_environment=None, + environment=None, working_directory=working_directory, borg_local_path=repo_create_command[0], borg_exit_codes=borg_exit_codes, diff --git a/tests/unit/borg/test_repo_delete.py b/tests/unit/borg/test_repo_delete.py index d490fa5d..66e73778 100644 --- a/tests/unit/borg/test_repo_delete.py +++ b/tests/unit/borg/test_repo_delete.py @@ -290,7 +290,7 @@ def test_delete_repository_with_defaults_does_not_capture_output(): command, output_log_level=module.logging.ANSWER, output_file=module.borgmatic.execute.DO_NOT_CAPTURE, - extra_environment=object, + environment=object, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -319,7 +319,7 @@ def test_delete_repository_with_force_captures_output(): command, output_log_level=module.logging.ANSWER, output_file=None, - extra_environment=object, + environment=object, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -348,7 +348,7 @@ def test_delete_repository_with_cache_only_captures_output(): command, output_log_level=module.logging.ANSWER, output_file=None, - extra_environment=object, + environment=object, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -379,7 +379,7 @@ def test_delete_repository_calls_borg_with_working_directory(): command, output_log_level=module.logging.ANSWER, output_file=module.borgmatic.execute.DO_NOT_CAPTURE, - extra_environment=object, + environment=object, working_directory='/working/dir', borg_local_path='borg', borg_exit_codes=None, diff --git a/tests/unit/borg/test_repo_info.py b/tests/unit/borg/test_repo_info.py index 6a0cec11..555cb566 100644 --- a/tests/unit/borg/test_repo_info.py +++ b/tests/unit/borg/test_repo_info.py @@ -24,7 +24,7 @@ def test_display_repository_info_calls_borg_with_flags(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -33,7 +33,7 @@ def test_display_repository_info_calls_borg_with_flags(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'repo-info', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -60,7 +60,7 @@ def test_display_repository_info_without_borg_features_calls_borg_with_info_sub_ flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--json', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -69,7 +69,7 @@ def test_display_repository_info_without_borg_features_calls_borg_with_info_sub_ flexmock(module).should_receive('execute_command').with_args( ('borg', 'info', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -101,7 +101,7 @@ def test_display_repository_info_with_log_info_calls_borg_with_info_flag(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--info', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -110,7 +110,7 @@ def test_display_repository_info_with_log_info_calls_borg_with_info_flag(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'repo-info', '--info', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -142,7 +142,7 @@ def test_display_repository_info_with_log_info_and_json_suppresses_most_borg_out flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -178,7 +178,7 @@ def test_display_repository_info_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--debug', '--show-rc', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -187,7 +187,7 @@ def test_display_repository_info_with_log_debug_calls_borg_with_debug_flag(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'repo-info', '--debug', '--show-rc', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -220,7 +220,7 @@ def test_display_repository_info_with_log_debug_and_json_suppresses_most_borg_ou flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -256,7 +256,7 @@ def test_display_repository_info_with_json_calls_borg_with_json_flag(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -291,7 +291,7 @@ def test_display_repository_info_with_local_path_calls_borg_via_local_path(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg1', 'repo-info', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -300,7 +300,7 @@ def test_display_repository_info_with_local_path_calls_borg_via_local_path(): flexmock(module).should_receive('execute_command').with_args( ('borg1', 'repo-info', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg1', borg_exit_codes=None, @@ -334,7 +334,7 @@ def test_display_repository_info_with_exit_codes_calls_borg_using_them(): borg_exit_codes = flexmock() flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=borg_exit_codes, @@ -343,7 +343,7 @@ def test_display_repository_info_with_exit_codes_calls_borg_using_them(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'repo-info', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=borg_exit_codes, @@ -375,7 +375,7 @@ def test_display_repository_info_with_remote_path_calls_borg_with_remote_path_fl flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--remote-path', 'borg1', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -384,7 +384,7 @@ def test_display_repository_info_with_remote_path_calls_borg_with_remote_path_fl flexmock(module).should_receive('execute_command').with_args( ('borg', 'repo-info', '--remote-path', 'borg1', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -417,7 +417,7 @@ def test_display_repository_info_with_umask_calls_borg_with_umask_flags(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--umask', '077', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -426,7 +426,7 @@ def test_display_repository_info_with_umask_calls_borg_with_umask_flags(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'repo-info', '--umask', '077', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -462,7 +462,7 @@ def test_display_repository_info_with_log_json_calls_borg_with_log_json_flags(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--log-json', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -471,7 +471,7 @@ def test_display_repository_info_with_log_json_calls_borg_with_log_json_flags(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'repo-info', '--log-json', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -504,7 +504,7 @@ def test_display_repository_info_with_lock_wait_calls_borg_with_lock_wait_flags( flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--lock-wait', '5', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -513,7 +513,7 @@ def test_display_repository_info_with_lock_wait_calls_borg_with_lock_wait_flags( flexmock(module).should_receive('execute_command').with_args( ('borg', 'repo-info', '--lock-wait', '5', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -547,7 +547,7 @@ def test_display_repository_info_calls_borg_with_working_directory(): ) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory='/working/dir', borg_local_path='borg', borg_exit_codes=None, @@ -556,7 +556,7 @@ def test_display_repository_info_calls_borg_with_working_directory(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'repo-info', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory='/working/dir', borg_local_path='borg', borg_exit_codes=None, diff --git a/tests/unit/borg/test_repo_list.py b/tests/unit/borg/test_repo_list.py index fec5e37e..da750d7a 100644 --- a/tests/unit/borg/test_repo_list.py +++ b/tests/unit/borg/test_repo_list.py @@ -39,7 +39,7 @@ def test_resolve_archive_name_calls_borg_with_flags(): ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, borg_local_path='borg', borg_exit_codes=None, - extra_environment=None, + environment=None, working_directory=None, ).and_return(expected_archive + '\n') @@ -61,7 +61,7 @@ def test_resolve_archive_name_with_log_info_calls_borg_without_info_flag(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -86,7 +86,7 @@ def test_resolve_archive_name_with_log_debug_calls_borg_without_debug_flag(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -111,7 +111,7 @@ def test_resolve_archive_name_with_local_path_calls_borg_via_local_path(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg1', 'list') + BORG_LIST_LATEST_ARGUMENTS, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg1', borg_exit_codes=None, @@ -137,7 +137,7 @@ def test_resolve_archive_name_with_exit_codes_calls_borg_using_them(): borg_exit_codes = flexmock() flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=borg_exit_codes, @@ -161,7 +161,7 @@ def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_flags flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list', '--remote-path', 'borg1') + BORG_LIST_LATEST_ARGUMENTS, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -186,7 +186,7 @@ def test_resolve_archive_name_with_umask_calls_borg_with_umask_flags(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list', '--umask', '077') + BORG_LIST_LATEST_ARGUMENTS, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -209,7 +209,7 @@ def test_resolve_archive_name_without_archives_raises(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -232,7 +232,7 @@ def test_resolve_archive_name_with_log_json_calls_borg_with_log_json_flags(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list', '--log-json') + BORG_LIST_LATEST_ARGUMENTS, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -257,7 +257,7 @@ def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_flags(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list', '--lock-wait', 'okay') + BORG_LIST_LATEST_ARGUMENTS, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -285,7 +285,7 @@ def test_resolve_archive_name_calls_borg_with_working_directory(): ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, borg_local_path='borg', borg_exit_codes=None, - extra_environment=None, + environment=None, working_directory='/working/dir', ).and_return(expected_archive + '\n') @@ -773,7 +773,7 @@ def test_list_repository_calls_borg_with_working_directory(): ) flexmock(module).should_receive('execute_command_and_capture_output').with_args( full_command=object, - extra_environment=object, + environment=object, working_directory='/working/dir', borg_local_path=object, borg_exit_codes=object, @@ -782,7 +782,7 @@ def test_list_repository_calls_borg_with_working_directory(): flexmock(module).should_receive('execute_command').with_args( full_command=object, output_log_level=object, - extra_environment=object, + environment=object, working_directory='/working/dir', borg_local_path=object, borg_exit_codes=object, diff --git a/tests/unit/borg/test_transfer.py b/tests/unit/borg/test_transfer.py index 49514861..edd2131b 100644 --- a/tests/unit/borg/test_transfer.py +++ b/tests/unit/borg/test_transfer.py @@ -21,7 +21,7 @@ def test_transfer_archives_calls_borg_with_flags(): ('borg', 'transfer', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -55,7 +55,7 @@ def test_transfer_archives_with_dry_run_calls_borg_with_dry_run_flag(): ('borg', 'transfer', '--repo', 'repo', '--dry-run'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -86,7 +86,7 @@ def test_transfer_archives_with_log_info_calls_borg_with_info_flag(): ('borg', 'transfer', '--info', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -117,7 +117,7 @@ def test_transfer_archives_with_log_debug_calls_borg_with_debug_flag(): ('borg', 'transfer', '--debug', '--show-rc', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -151,7 +151,7 @@ def test_transfer_archives_with_archive_calls_borg_with_match_archives_flag(): ('borg', 'transfer', '--match-archives', 'archive', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -184,7 +184,7 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl ('borg', 'transfer', '--match-archives', 'sh:foo*', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -217,7 +217,7 @@ def test_transfer_archives_with_archive_name_format_calls_borg_with_match_archiv ('borg', 'transfer', '--match-archives', 'sh:bar-*', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -248,7 +248,7 @@ def test_transfer_archives_with_local_path_calls_borg_via_local_path(): ('borg2', 'transfer', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg2', borg_exit_codes=None, @@ -281,7 +281,7 @@ def test_transfer_archives_with_exit_codes_calls_borg_using_them(): ('borg', 'transfer', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=borg_exit_codes, @@ -315,7 +315,7 @@ def test_transfer_archives_with_remote_path_calls_borg_with_remote_path_flags(): ('borg', 'transfer', '--remote-path', 'borg2', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -349,7 +349,7 @@ def test_transfer_archives_with_umask_calls_borg_with_umask_flags(): ('borg', 'transfer', '--umask', '077', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -383,7 +383,7 @@ def test_transfer_archives_with_log_json_calls_borg_with_log_json_flags(): ('borg', 'transfer', '--log-json', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -418,7 +418,7 @@ def test_transfer_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): ('borg', 'transfer', '--lock-wait', '5', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -449,7 +449,7 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): ('borg', 'transfer', '--progress', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=module.DO_NOT_CAPTURE, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -484,7 +484,7 @@ def test_transfer_archives_passes_through_arguments_to_borg(argument_name): ('borg', 'transfer', flag_name, 'value', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -521,7 +521,7 @@ def test_transfer_archives_with_source_repository_calls_borg_with_other_repo_fla ('borg', 'transfer', '--repo', 'repo', '--other-repo', 'other'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -566,7 +566,7 @@ def test_transfer_archives_with_date_based_matching_calls_borg_with_date_based_f ), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -605,7 +605,7 @@ def test_transfer_archives_calls_borg_with_working_directory(): ('borg', 'transfer', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory='/working/dir', borg_local_path='borg', borg_exit_codes=None, diff --git a/tests/unit/borg/test_version.py b/tests/unit/borg/test_version.py index 383eff55..17fc1861 100644 --- a/tests/unit/borg/test_version.py +++ b/tests/unit/borg/test_version.py @@ -23,7 +23,7 @@ def insert_execute_command_and_capture_output_mock( ) flexmock(module).should_receive('execute_command_and_capture_output').with_args( command, - extra_environment=None, + environment=None, working_directory=working_directory, borg_local_path=borg_local_path, borg_exit_codes=borg_exit_codes, diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py index 30209f51..474c7d6d 100644 --- a/tests/unit/hooks/data_source/test_mariadb.py +++ b/tests/unit/hooks/data_source/test_mariadb.py @@ -7,33 +7,33 @@ from borgmatic.hooks.data_source import mariadb as module def test_database_names_to_dump_passes_through_name(): - extra_environment = flexmock() + environment = flexmock() - names = module.database_names_to_dump({'name': 'foo'}, {}, extra_environment, dry_run=False) + names = module.database_names_to_dump({'name': 'foo'}, {}, environment, dry_run=False) assert names == ('foo',) def test_database_names_to_dump_bails_for_dry_run(): - extra_environment = flexmock() + environment = flexmock() flexmock(module).should_receive('execute_command_and_capture_output').never() - names = module.database_names_to_dump({'name': 'all'}, {}, extra_environment, dry_run=True) + names = module.database_names_to_dump({'name': 'all'}, {}, environment, dry_run=True) assert names == () def test_database_names_to_dump_queries_mariadb_for_database_names(): - extra_environment = flexmock() + environment = flexmock() flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('mariadb', '--skip-column-names', '--batch', '--execute', 'show schemas'), - extra_environment=extra_environment, + environment=environment, ).and_return('foo\nbar\nmysql\n').once() - names = module.database_names_to_dump({'name': 'all'}, {}, extra_environment, dry_run=False) + names = module.database_names_to_dump({'name': 'all'}, {}, environment, dry_run=False) assert names == ('foo', 'bar') @@ -53,6 +53,7 @@ def test_dump_data_sources_dumps_each_database(): databases = [{'name': 'foo'}, {'name': 'bar'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) @@ -66,7 +67,7 @@ def test_dump_data_sources_dumps_each_database(): config={}, dump_path=object, database_names=(name,), - extra_environment=object, + environment={'USER': 'root'}, dry_run=object, dry_run_label=object, ).and_return(process).once() @@ -88,6 +89,7 @@ def test_dump_data_sources_dumps_with_password(): database = {'name': 'foo', 'username': 'root', 'password': 'trustsome1'} process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) @@ -100,7 +102,7 @@ def test_dump_data_sources_dumps_with_password(): config={}, dump_path=object, database_names=('foo',), - extra_environment={'MYSQL_PWD': 'trustsome1'}, + environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'}, dry_run=object, dry_run_label=object, ).and_return(process).once() @@ -119,6 +121,7 @@ def test_dump_data_sources_dumps_all_databases_at_once(): databases = [{'name': 'all'}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) @@ -128,7 +131,7 @@ def test_dump_data_sources_dumps_all_databases_at_once(): config={}, dump_path=object, database_names=('foo', 'bar'), - extra_environment=object, + environment={'USER': 'root'}, dry_run=object, dry_run_label=object, ).and_return(process).once() @@ -147,6 +150,7 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured databases = [{'name': 'all', 'format': 'sql'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) @@ -158,7 +162,7 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured config={}, dump_path=object, database_names=(name,), - extra_environment=object, + environment={'USER': 'root'}, dry_run=object, dry_run_label=object, ).and_return(process).once() @@ -187,7 +191,7 @@ def test_database_names_to_dump_runs_mariadb_with_list_options(): '--execute', 'show schemas', ), - extra_environment=None, + environment=None, ).and_return(('foo\nbar')).once() assert module.database_names_to_dump(database, {}, None, '') == ('foo', 'bar') @@ -200,7 +204,7 @@ def test_database_names_to_dump_runs_non_default_mariadb_with_list_options(): 'mariadb_command': 'custom_mariadb', } flexmock(module).should_receive('execute_command_and_capture_output').with_args( - extra_environment=None, + environment=None, full_command=( 'custom_mariadb', # Custom MariaDB command '--defaults-extra-file=mariadb.cnf', @@ -232,7 +236,7 @@ def test_execute_dump_command_runs_mariadb_dump(): '--result-file', 'dump', ), - extra_environment=None, + environment=None, run_to_completion=False, ).and_return(process).once() @@ -242,7 +246,7 @@ def test_execute_dump_command_runs_mariadb_dump(): config={}, dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=False, dry_run_label='', ) @@ -267,7 +271,7 @@ def test_execute_dump_command_runs_mariadb_dump_without_add_drop_database(): '--result-file', 'dump', ), - extra_environment=None, + environment=None, run_to_completion=False, ).and_return(process).once() @@ -277,7 +281,7 @@ def test_execute_dump_command_runs_mariadb_dump_without_add_drop_database(): config={}, dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=False, dry_run_label='', ) @@ -309,7 +313,7 @@ def test_execute_dump_command_runs_mariadb_dump_with_hostname_and_port(): '--result-file', 'dump', ), - extra_environment=None, + environment=None, run_to_completion=False, ).and_return(process).once() @@ -319,7 +323,7 @@ def test_execute_dump_command_runs_mariadb_dump_with_hostname_and_port(): config={}, dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=False, dry_run_label='', ) @@ -347,7 +351,7 @@ def test_execute_dump_command_runs_mariadb_dump_with_username_and_password(): '--result-file', 'dump', ), - extra_environment={'MYSQL_PWD': 'trustsome1'}, + environment={'MYSQL_PWD': 'trustsome1'}, run_to_completion=False, ).and_return(process).once() @@ -357,7 +361,7 @@ def test_execute_dump_command_runs_mariadb_dump_with_username_and_password(): config={}, dump_path=flexmock(), database_names=('foo',), - extra_environment={'MYSQL_PWD': 'trustsome1'}, + environment={'MYSQL_PWD': 'trustsome1'}, dry_run=False, dry_run_label='', ) @@ -384,7 +388,7 @@ def test_execute_dump_command_runs_mariadb_dump_with_options(): '--result-file', 'dump', ), - extra_environment=None, + environment=None, run_to_completion=False, ).and_return(process).once() @@ -394,7 +398,7 @@ def test_execute_dump_command_runs_mariadb_dump_with_options(): config={}, dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=False, dry_run_label='', ) @@ -421,7 +425,7 @@ def test_execute_dump_command_runs_non_default_mariadb_dump_with_options(): '--result-file', 'dump', ), - extra_environment=None, + environment=None, run_to_completion=False, ).and_return(process).once() @@ -435,7 +439,7 @@ def test_execute_dump_command_runs_non_default_mariadb_dump_with_options(): config={}, dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=False, dry_run_label='', ) @@ -455,7 +459,7 @@ def test_execute_dump_command_with_duplicate_dump_skips_mariadb_dump(): config={}, dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=True, dry_run_label='SO DRY', ) @@ -479,7 +483,7 @@ def test_execute_dump_command_with_dry_run_skips_mariadb_dump(): config={}, dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=True, dry_run_label='SO DRY', ) @@ -490,6 +494,7 @@ def test_execute_dump_command_with_dry_run_skips_mariadb_dump(): def test_dump_data_sources_errors_for_missing_all_databases(): databases = [{'name': 'all'}] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) @@ -512,6 +517,7 @@ def test_dump_data_sources_errors_for_missing_all_databases(): def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run(): databases = [{'name': 'all'}] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) @@ -540,12 +546,13 @@ def test_restore_data_source_dump_runs_mariadb_to_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mariadb', '--batch'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment=None, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -571,12 +578,13 @@ def test_restore_data_source_dump_runs_mariadb_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mariadb', '--batch', '--harder'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment=None, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -604,12 +612,13 @@ def test_restore_data_source_dump_runs_non_default_mariadb_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('custom_mariadb', '--batch', '--harder'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment=None, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -635,6 +644,7 @@ def test_restore_data_source_dump_runs_mariadb_with_hostname_and_port(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mariadb', @@ -649,7 +659,7 @@ def test_restore_data_source_dump_runs_mariadb_with_hostname_and_port(): processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment=None, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -675,12 +685,13 @@ def test_restore_data_source_dump_runs_mariadb_with_username_and_password(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mariadb', '--batch', '--user', 'root'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'MYSQL_PWD': 'trustsome1'}, + environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'}, ).once() module.restore_data_source_dump( @@ -716,6 +727,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mariadb', @@ -732,7 +744,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'MYSQL_PWD': 'clipassword'}, + environment={'USER': 'root', 'MYSQL_PWD': 'clipassword'}, ).once() module.restore_data_source_dump( @@ -770,6 +782,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mariadb', @@ -786,7 +799,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'MYSQL_PWD': 'restorepass'}, + environment={'USER': 'root', 'MYSQL_PWD': 'restorepass'}, ).once() module.restore_data_source_dump( @@ -811,6 +824,7 @@ def test_restore_data_source_dump_with_dry_run_skips_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').never() module.restore_data_source_dump( diff --git a/tests/unit/hooks/data_source/test_mysql.py b/tests/unit/hooks/data_source/test_mysql.py index 3fcf43eb..5b4cbcc9 100644 --- a/tests/unit/hooks/data_source/test_mysql.py +++ b/tests/unit/hooks/data_source/test_mysql.py @@ -7,36 +7,36 @@ from borgmatic.hooks.data_source import mysql as module def test_database_names_to_dump_passes_through_name(): - extra_environment = flexmock() + environment = flexmock() - names = module.database_names_to_dump({'name': 'foo'}, {}, extra_environment, dry_run=False) + names = module.database_names_to_dump({'name': 'foo'}, {}, environment, dry_run=False) assert names == ('foo',) def test_database_names_to_dump_bails_for_dry_run(): - extra_environment = flexmock() + environment = flexmock() flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_and_capture_output').never() - names = module.database_names_to_dump({'name': 'all'}, {}, extra_environment, dry_run=True) + names = module.database_names_to_dump({'name': 'all'}, {}, environment, dry_run=True) assert names == () def test_database_names_to_dump_queries_mysql_for_database_names(): - extra_environment = flexmock() + environment = flexmock() flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('mysql', '--skip-column-names', '--batch', '--execute', 'show schemas'), - extra_environment=extra_environment, + environment=environment, ).and_return('foo\nbar\nmysql\n').once() - names = module.database_names_to_dump({'name': 'all'}, {}, extra_environment, dry_run=False) + names = module.database_names_to_dump({'name': 'all'}, {}, environment, dry_run=False) assert names == ('foo', 'bar') @@ -56,6 +56,7 @@ def test_dump_data_sources_dumps_each_database(): databases = [{'name': 'foo'}, {'name': 'bar'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( ('bar',) ) @@ -66,7 +67,7 @@ def test_dump_data_sources_dumps_each_database(): config={}, dump_path=object, database_names=(name,), - extra_environment=object, + environment={'USER': 'root'}, dry_run=object, dry_run_label=object, ).and_return(process).once() @@ -88,6 +89,7 @@ def test_dump_data_sources_dumps_with_password(): database = {'name': 'foo', 'username': 'root', 'password': 'trustsome1'} process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) @@ -100,7 +102,7 @@ def test_dump_data_sources_dumps_with_password(): config={}, dump_path=object, database_names=('foo',), - extra_environment={'MYSQL_PWD': 'trustsome1'}, + environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'}, dry_run=object, dry_run_label=object, ).and_return(process).once() @@ -119,13 +121,14 @@ def test_dump_data_sources_dumps_all_databases_at_once(): databases = [{'name': 'all'}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) flexmock(module).should_receive('execute_dump_command').with_args( database={'name': 'all'}, config={}, dump_path=object, database_names=('foo', 'bar'), - extra_environment=object, + environment={'USER': 'root'}, dry_run=object, dry_run_label=object, ).and_return(process).once() @@ -144,6 +147,7 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured databases = [{'name': 'all', 'format': 'sql'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) for name, process in zip(('foo', 'bar'), processes): @@ -152,7 +156,7 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured config={}, dump_path=object, database_names=(name,), - extra_environment=object, + environment={'USER': 'root'}, dry_run=object, dry_run_label=object, ).and_return(process).once() @@ -181,7 +185,7 @@ def test_database_names_to_dump_runs_mysql_with_list_options(): '--execute', 'show schemas', ), - extra_environment=None, + environment=None, ).and_return(('foo\nbar')).once() assert module.database_names_to_dump(database, {}, None, '') == ('foo', 'bar') @@ -194,7 +198,7 @@ def test_database_names_to_dump_runs_non_default_mysql_with_list_options(): 'mysql_command': 'custom_mysql', } flexmock(module).should_receive('execute_command_and_capture_output').with_args( - extra_environment=None, + environment=None, full_command=( 'custom_mysql', # Custom MySQL command '--defaults-extra-file=my.cnf', @@ -226,7 +230,7 @@ def test_execute_dump_command_runs_mysqldump(): '--result-file', 'dump', ), - extra_environment=None, + environment=None, run_to_completion=False, ).and_return(process).once() @@ -236,7 +240,7 @@ def test_execute_dump_command_runs_mysqldump(): config={}, dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=False, dry_run_label='', ) @@ -261,7 +265,7 @@ def test_execute_dump_command_runs_mysqldump_without_add_drop_database(): '--result-file', 'dump', ), - extra_environment=None, + environment=None, run_to_completion=False, ).and_return(process).once() @@ -271,7 +275,7 @@ def test_execute_dump_command_runs_mysqldump_without_add_drop_database(): config={}, dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=False, dry_run_label='', ) @@ -303,7 +307,7 @@ def test_execute_dump_command_runs_mysqldump_with_hostname_and_port(): '--result-file', 'dump', ), - extra_environment=None, + environment=None, run_to_completion=False, ).and_return(process).once() @@ -313,7 +317,7 @@ def test_execute_dump_command_runs_mysqldump_with_hostname_and_port(): config={}, dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=False, dry_run_label='', ) @@ -341,7 +345,7 @@ def test_execute_dump_command_runs_mysqldump_with_username_and_password(): '--result-file', 'dump', ), - extra_environment={'MYSQL_PWD': 'trustsome1'}, + environment={'MYSQL_PWD': 'trustsome1'}, run_to_completion=False, ).and_return(process).once() @@ -351,7 +355,7 @@ def test_execute_dump_command_runs_mysqldump_with_username_and_password(): config={}, dump_path=flexmock(), database_names=('foo',), - extra_environment={'MYSQL_PWD': 'trustsome1'}, + environment={'MYSQL_PWD': 'trustsome1'}, dry_run=False, dry_run_label='', ) @@ -378,7 +382,7 @@ def test_execute_dump_command_runs_mysqldump_with_options(): '--result-file', 'dump', ), - extra_environment=None, + environment=None, run_to_completion=False, ).and_return(process).once() @@ -388,7 +392,7 @@ def test_execute_dump_command_runs_mysqldump_with_options(): config={}, dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=False, dry_run_label='', ) @@ -414,7 +418,7 @@ def test_execute_dump_command_runs_non_default_mysqldump(): '--result-file', 'dump', ), - extra_environment=None, + environment=None, run_to_completion=False, ).and_return(process).once() @@ -427,7 +431,7 @@ def test_execute_dump_command_runs_non_default_mysqldump(): config={}, dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=False, dry_run_label='', ) @@ -447,7 +451,7 @@ def test_execute_dump_command_with_duplicate_dump_skips_mysqldump(): config={}, dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=True, dry_run_label='SO DRY', ) @@ -471,7 +475,7 @@ def test_execute_dump_command_with_dry_run_skips_mysqldump(): config={}, dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=True, dry_run_label='SO DRY', ) @@ -482,6 +486,7 @@ def test_execute_dump_command_with_dry_run_skips_mysqldump(): def test_dump_data_sources_errors_for_missing_all_databases(): databases = [{'name': 'all'}] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) @@ -504,6 +509,7 @@ def test_dump_data_sources_errors_for_missing_all_databases(): def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run(): databases = [{'name': 'all'}] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) @@ -532,12 +538,13 @@ def test_restore_data_source_dump_runs_mysql_to_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mysql', '--batch'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment=None, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -563,12 +570,13 @@ def test_restore_data_source_dump_runs_mysql_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mysql', '--batch', '--harder'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment=None, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -594,12 +602,13 @@ def test_restore_data_source_dump_runs_non_default_mysql_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('custom_mysql', '--batch', '--harder'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment=None, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -625,6 +634,7 @@ def test_restore_data_source_dump_runs_mysql_with_hostname_and_port(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mysql', @@ -639,7 +649,7 @@ def test_restore_data_source_dump_runs_mysql_with_hostname_and_port(): processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment=None, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -665,12 +675,13 @@ def test_restore_data_source_dump_runs_mysql_with_username_and_password(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mysql', '--batch', '--user', 'root'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'MYSQL_PWD': 'trustsome1'}, + environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'}, ).once() module.restore_data_source_dump( @@ -706,6 +717,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mysql', @@ -722,7 +734,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'MYSQL_PWD': 'clipassword'}, + environment={'USER': 'root', 'MYSQL_PWD': 'clipassword'}, ).once() module.restore_data_source_dump( @@ -760,6 +772,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mysql', @@ -776,7 +789,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'MYSQL_PWD': 'restorepass'}, + environment={'USER': 'root', 'MYSQL_PWD': 'restorepass'}, ).once() module.restore_data_source_dump( @@ -801,6 +814,7 @@ def test_restore_data_source_dump_with_dry_run_skips_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').never() module.restore_data_source_dump( diff --git a/tests/unit/hooks/data_source/test_postgresql.py b/tests/unit/hooks/data_source/test_postgresql.py index 6bdf083c..1037cf66 100644 --- a/tests/unit/hooks/data_source/test_postgresql.py +++ b/tests/unit/hooks/data_source/test_postgresql.py @@ -6,7 +6,7 @@ from flexmock import flexmock from borgmatic.hooks.data_source import postgresql as module -def test_make_extra_environment_maps_options_to_environment(): +def test_make_environment_maps_options_to_environment(): database = { 'name': 'foo', 'password': 'pass', @@ -17,6 +17,7 @@ def test_make_extra_environment_maps_options_to_environment(): 'ssl_crl': 'crl.crl', } expected = { + 'USER': 'root', 'PGPASSWORD': 'pass', 'PGSSLMODE': 'require', 'PGSSLCERT': 'cert.crt', @@ -24,44 +25,44 @@ def test_make_extra_environment_maps_options_to_environment(): 'PGSSLROOTCERT': 'root.crt', 'PGSSLCRL': 'crl.crl', } + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - extra_env = module.make_extra_environment(database, {}) - - assert extra_env == expected + assert module.make_environment(database, {}) == expected -def test_make_extra_environment_with_cli_password_sets_correct_password(): +def test_make_environment_with_cli_password_sets_correct_password(): database = {'name': 'foo', 'restore_password': 'trustsome1', 'password': 'anotherpassword'} + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - extra = module.make_extra_environment( + environment = module.make_environment( database, {}, restore_connection_params={'password': 'clipassword'} ) - assert extra['PGPASSWORD'] == 'clipassword' + assert environment['PGPASSWORD'] == 'clipassword' -def test_make_extra_environment_without_cli_password_or_configured_password_does_not_set_password(): +def test_make_environment_without_cli_password_or_configured_password_does_not_set_password(): database = {'name': 'foo'} - extra = module.make_extra_environment( + environment = module.make_environment( database, {}, restore_connection_params={'username': 'someone'} ) - assert 'PGPASSWORD' not in extra + assert 'PGPASSWORD' not in environment -def test_make_extra_environment_without_ssl_mode_does_not_set_ssl_mode(): +def test_make_environment_without_ssl_mode_does_not_set_ssl_mode(): database = {'name': 'foo'} - extra = module.make_extra_environment(database, {}) + environment = module.make_environment(database, {}) - assert 'PGSSLMODE' not in extra + assert 'PGSSLMODE' not in environment def test_database_names_to_dump_passes_through_individual_database_name(): @@ -125,7 +126,7 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostnam '--port', '1234', ), - extra_environment=object, + environment=object, ).and_return('foo,test,\nbar,test,"stuff and such"') assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ( @@ -150,7 +151,7 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_usernam '--username', 'postgres', ), - extra_environment=object, + environment=object, ).and_return('foo,test,\nbar,test,"stuff and such"') assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ( @@ -166,7 +167,7 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_options ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('psql', '--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only', '--harder'), - extra_environment=object, + environment=object, ).and_return('foo,test,\nbar,test,"stuff and such"') assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ( @@ -210,7 +211,7 @@ def test_database_names_to_dump_with_all_and_psql_command_uses_custom_command(): '--csv', '--tuples-only', ), - extra_environment=object, + environment=object, ).and_return('foo,text').once() assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('foo',) @@ -237,7 +238,7 @@ def test_use_streaming_false_for_no_databases(): def test_dump_data_sources_runs_pg_dump_for_each_database(): databases = [{'name': 'foo'}, {'name': 'bar'}] processes = [flexmock(), flexmock()] - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( ('bar',) @@ -265,7 +266,7 @@ def test_dump_data_sources_runs_pg_dump_for_each_database(): f'databases/localhost/{name}', ), shell=True, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() @@ -284,7 +285,7 @@ def test_dump_data_sources_runs_pg_dump_for_each_database(): def test_dump_data_sources_raises_when_no_database_names_to_dump(): databases = [{'name': 'foo'}, {'name': 'bar'}] - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module).should_receive('database_names_to_dump').and_return(()) @@ -301,7 +302,7 @@ def test_dump_data_sources_raises_when_no_database_names_to_dump(): def test_dump_data_sources_does_not_raise_when_no_database_names_to_dump(): databases = [{'name': 'foo'}, {'name': 'bar'}] - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module).should_receive('database_names_to_dump').and_return(()) @@ -317,7 +318,7 @@ def test_dump_data_sources_does_not_raise_when_no_database_names_to_dump(): def test_dump_data_sources_with_duplicate_dump_skips_pg_dump(): databases = [{'name': 'foo'}, {'name': 'bar'}] - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( ('bar',) @@ -344,7 +345,7 @@ def test_dump_data_sources_with_duplicate_dump_skips_pg_dump(): def test_dump_data_sources_with_dry_run_skips_pg_dump(): databases = [{'name': 'foo'}, {'name': 'bar'}] - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( ('bar',) @@ -375,7 +376,7 @@ def test_dump_data_sources_with_dry_run_skips_pg_dump(): def test_dump_data_sources_runs_pg_dump_with_hostname_and_port(): databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] process = flexmock() - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( @@ -404,7 +405,7 @@ def test_dump_data_sources_runs_pg_dump_with_hostname_and_port(): 'databases/database.example.org/foo', ), shell=True, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() @@ -421,7 +422,7 @@ def test_dump_data_sources_runs_pg_dump_with_hostname_and_port(): def test_dump_data_sources_runs_pg_dump_with_username_and_password(): databases = [{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1'}] process = flexmock() - flexmock(module).should_receive('make_extra_environment').and_return( + flexmock(module).should_receive('make_environment').and_return( {'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'} ) flexmock(module).should_receive('make_dump_path').and_return('') @@ -450,7 +451,7 @@ def test_dump_data_sources_runs_pg_dump_with_username_and_password(): 'databases/localhost/foo', ), shell=True, - extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}, + environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() @@ -467,7 +468,7 @@ def test_dump_data_sources_runs_pg_dump_with_username_and_password(): def test_dump_data_sources_with_username_injection_attack_gets_escaped(): databases = [{'name': 'foo', 'username': 'postgres; naughty-command', 'password': 'trustsome1'}] process = flexmock() - flexmock(module).should_receive('make_extra_environment').and_return( + flexmock(module).should_receive('make_environment').and_return( {'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'} ) flexmock(module).should_receive('make_dump_path').and_return('') @@ -496,7 +497,7 @@ def test_dump_data_sources_with_username_injection_attack_gets_escaped(): 'databases/localhost/foo', ), shell=True, - extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}, + environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() @@ -512,7 +513,7 @@ def test_dump_data_sources_with_username_injection_attack_gets_escaped(): def test_dump_data_sources_runs_pg_dump_with_directory_format(): databases = [{'name': 'foo', 'format': 'directory'}] - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( @@ -538,7 +539,7 @@ def test_dump_data_sources_runs_pg_dump_with_directory_format(): 'foo', ), shell=True, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).and_return(flexmock()).once() assert ( @@ -557,7 +558,7 @@ def test_dump_data_sources_runs_pg_dump_with_directory_format(): def test_dump_data_sources_runs_pg_dump_with_options(): databases = [{'name': 'foo', 'options': '--stuff=such'}] process = flexmock() - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( @@ -583,7 +584,7 @@ def test_dump_data_sources_runs_pg_dump_with_options(): 'databases/localhost/foo', ), shell=True, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() @@ -600,7 +601,7 @@ def test_dump_data_sources_runs_pg_dump_with_options(): def test_dump_data_sources_runs_pg_dumpall_for_all_databases(): databases = [{'name': 'all'}] process = flexmock() - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module).should_receive('database_names_to_dump').and_return(('all',)) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( @@ -615,7 +616,7 @@ def test_dump_data_sources_runs_pg_dumpall_for_all_databases(): flexmock(module).should_receive('execute_command').with_args( ('pg_dumpall', '--no-password', '--clean', '--if-exists', '>', 'databases/localhost/all'), shell=True, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() @@ -632,7 +633,7 @@ def test_dump_data_sources_runs_pg_dumpall_for_all_databases(): def test_dump_data_sources_runs_non_default_pg_dump(): databases = [{'name': 'foo', 'pg_dump_command': 'special_pg_dump --compress *'}] process = flexmock() - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( @@ -659,7 +660,7 @@ def test_dump_data_sources_runs_non_default_pg_dump(): 'databases/localhost/foo', ), shell=True, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() @@ -680,7 +681,7 @@ def test_restore_data_source_dump_runs_pg_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -696,7 +697,7 @@ def test_restore_data_source_dump_runs_pg_restore(): processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( ( @@ -709,7 +710,7 @@ def test_restore_data_source_dump_runs_pg_restore(): '--command', 'ANALYZE', ), - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() module.restore_data_source_dump( @@ -737,7 +738,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_hostname_and_port(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -757,7 +758,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_hostname_and_port(): processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( ( @@ -774,7 +775,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_hostname_and_port(): '--command', 'ANALYZE', ), - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() module.restore_data_source_dump( @@ -802,7 +803,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_username_and_password(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_extra_environment').and_return( + flexmock(module).should_receive('make_environment').and_return( {'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'} ) flexmock(module).should_receive('make_dump_path') @@ -822,7 +823,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_username_and_password(): processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}, + environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( ( @@ -837,7 +838,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_username_and_password(): '--command', 'ANALYZE', ), - extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}, + environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}, ).once() module.restore_data_source_dump( @@ -876,7 +877,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_extra_environment').and_return( + flexmock(module).should_receive('make_environment').and_return( {'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'} ) flexmock(module).should_receive('make_dump_path') @@ -900,7 +901,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'}, + environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( ( @@ -919,7 +920,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ '--command', 'ANALYZE', ), - extra_environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'}, + environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'}, ).once() module.restore_data_source_dump( @@ -958,7 +959,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_extra_environment').and_return( + flexmock(module).should_receive('make_environment').and_return( {'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'} ) flexmock(module).should_receive('make_dump_path') @@ -982,7 +983,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'}, + environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( ( @@ -1001,7 +1002,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ '--command', 'ANALYZE', ), - extra_environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'}, + environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'}, ).once() module.restore_data_source_dump( @@ -1034,7 +1035,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -1051,7 +1052,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_options(): processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( ( @@ -1065,7 +1066,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_options(): '--command', 'ANALYZE', ), - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() module.restore_data_source_dump( @@ -1091,7 +1092,7 @@ def test_restore_data_source_dump_runs_psql_for_all_database_dump(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -1103,11 +1104,11 @@ def test_restore_data_source_dump_runs_psql_for_all_database_dump(): processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( ('psql', '--no-password', '--no-psqlrc', '--quiet', '--command', 'ANALYZE'), - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() module.restore_data_source_dump( @@ -1133,7 +1134,7 @@ def test_restore_data_source_dump_runs_psql_for_plain_database_dump(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -1141,7 +1142,7 @@ def test_restore_data_source_dump_runs_psql_for_plain_database_dump(): processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( ( @@ -1154,7 +1155,7 @@ def test_restore_data_source_dump_runs_psql_for_plain_database_dump(): '--command', 'ANALYZE', ), - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() module.restore_data_source_dump( @@ -1187,7 +1188,7 @@ def test_restore_data_source_dump_runs_non_default_pg_restore_and_psql(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -1208,7 +1209,7 @@ def test_restore_data_source_dump_runs_non_default_pg_restore_and_psql(): processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( ( @@ -1226,7 +1227,7 @@ def test_restore_data_source_dump_runs_non_default_pg_restore_and_psql(): '--command', 'ANALYZE', ), - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() module.restore_data_source_dump( @@ -1251,7 +1252,7 @@ def test_restore_data_source_dump_with_dry_run_skips_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') flexmock(module).should_receive('execute_command_with_processes').never() @@ -1278,7 +1279,7 @@ def test_restore_data_source_dump_without_extract_process_restores_from_disk(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('/dump/path') flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -1295,7 +1296,7 @@ def test_restore_data_source_dump_without_extract_process_restores_from_disk(): processes=[], output_log_level=logging.DEBUG, input_file=None, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( ( @@ -1308,7 +1309,7 @@ def test_restore_data_source_dump_without_extract_process_restores_from_disk(): '--command', 'ANALYZE', ), - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() module.restore_data_source_dump( @@ -1333,7 +1334,7 @@ def test_restore_data_source_dump_with_schemas_restores_schemas(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('/dump/path') flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -1354,7 +1355,7 @@ def test_restore_data_source_dump_with_schemas_restores_schemas(): processes=[], output_log_level=logging.DEBUG, input_file=None, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( ( @@ -1367,7 +1368,7 @@ def test_restore_data_source_dump_with_schemas_restores_schemas(): '--command', 'ANALYZE', ), - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() module.restore_data_source_dump( diff --git a/tests/unit/hooks/test_command.py b/tests/unit/hooks/test_command.py index 7f2e7384..5d3d285a 100644 --- a/tests/unit/hooks/test_command.py +++ b/tests/unit/hooks/test_command.py @@ -45,7 +45,7 @@ def test_make_environment_with_pyinstaller_clears_LD_LIBRARY_PATH(): def test_make_environment_with_pyinstaller_and_LD_LIBRARY_PATH_ORIG_copies_it_into_LD_LIBRARY_PATH(): assert module.make_environment( {'LD_LIBRARY_PATH_ORIG': '/lib/lib/lib'}, sys_module=flexmock(frozen=True, _MEIPASS='yup') - ) == {'LD_LIBRARY_PATH': '/lib/lib/lib'} + ) == {'LD_LIBRARY_PATH_ORIG': '/lib/lib/lib', 'LD_LIBRARY_PATH': '/lib/lib/lib'} def test_execute_hook_invokes_each_command(): @@ -57,7 +57,7 @@ def test_execute_hook_invokes_each_command(): [':'], output_log_level=logging.WARNING, shell=True, - extra_environment={}, + environment={}, ).once() module.execute_hook([':'], None, 'config.yaml', 'pre-backup', dry_run=False) @@ -72,13 +72,13 @@ def test_execute_hook_with_multiple_commands_invokes_each_command(): [':'], output_log_level=logging.WARNING, shell=True, - extra_environment={}, + environment={}, ).once() flexmock(module.borgmatic.execute).should_receive('execute_command').with_args( ['true'], output_log_level=logging.WARNING, shell=True, - extra_environment={}, + environment={}, ).once() module.execute_hook([':', 'true'], None, 'config.yaml', 'pre-backup', dry_run=False) @@ -95,7 +95,7 @@ def test_execute_hook_with_umask_sets_that_umask(): [':'], output_log_level=logging.WARNING, shell=True, - extra_environment={}, + environment={}, ) module.execute_hook([':'], 77, 'config.yaml', 'pre-backup', dry_run=False) @@ -124,7 +124,7 @@ def test_execute_hook_on_error_logs_as_error(): [':'], output_log_level=logging.ERROR, shell=True, - extra_environment={}, + environment={}, ).once() module.execute_hook([':'], None, 'config.yaml', 'on-error', dry_run=False) diff --git a/tests/unit/test_execute.py b/tests/unit/test_execute.py index c9ca6e77..4d39fe3b 100644 --- a/tests/unit/test_execute.py +++ b/tests/unit/test_execute.py @@ -159,8 +159,15 @@ def test_mask_command_secrets_passes_through_other_commands(): ('foo', 'bar'), None, None, - {'DBPASS': 'secret', 'OTHER': 'thing'}, - 'DBPASS=*** OTHER=*** foo bar', + {'UNKNOWN': 'secret', 'OTHER': 'thing'}, + 'foo bar', + ), + ( + ('foo', 'bar'), + None, + None, + {'PGTHING': 'secret', 'BORG_OTHER': 'thing'}, + 'PGTHING=*** BORG_OTHER=*** foo bar', ), ), ) @@ -176,7 +183,6 @@ def test_log_command_logs_command_constructed_from_arguments( def test_execute_command_calls_full_command(): full_command = ['foo', 'bar'] flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, @@ -199,7 +205,6 @@ def test_execute_command_calls_full_command_with_output_file(): full_command = ['foo', 'bar'] output_file = flexmock(name='test') flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, @@ -221,7 +226,6 @@ def test_execute_command_calls_full_command_with_output_file(): def test_execute_command_calls_full_command_without_capturing_output(): full_command = ['foo', 'bar'] flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, @@ -245,7 +249,6 @@ def test_execute_command_calls_full_command_with_input_file(): full_command = ['foo', 'bar'] input_file = flexmock(name='test') flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=input_file, @@ -267,7 +270,6 @@ def test_execute_command_calls_full_command_with_input_file(): def test_execute_command_calls_full_command_with_shell(): full_command = ['foo', 'bar'] flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( ' '.join(full_command), stdin=None, @@ -286,24 +288,23 @@ def test_execute_command_calls_full_command_with_shell(): assert output is None -def test_execute_command_calls_full_command_with_extra_environment(): +def test_execute_command_calls_full_command_with_environment(): full_command = ['foo', 'bar'] flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, stdout=module.subprocess.PIPE, stderr=module.subprocess.STDOUT, shell=False, - env={'a': 'b', 'c': 'd'}, + env={'a': 'b'}, cwd=None, close_fds=True, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') - output = module.execute_command(full_command, extra_environment={'c': 'd'}) + output = module.execute_command(full_command, environment={'a': 'b'}) assert output is None @@ -311,7 +312,6 @@ def test_execute_command_calls_full_command_with_extra_environment(): def test_execute_command_calls_full_command_with_working_directory(): full_command = ['foo', 'bar'] flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, @@ -333,21 +333,20 @@ def test_execute_command_calls_full_command_with_working_directory(): def test_execute_command_with_BORG_PASSPHRASE_FD_leaves_file_descriptors_open(): full_command = ['foo', 'bar'] flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, stdout=module.subprocess.PIPE, stderr=module.subprocess.STDOUT, shell=False, - env={'a': 'b', 'BORG_PASSPHRASE_FD': '4'}, + env={'BORG_PASSPHRASE_FD': '4'}, cwd=None, close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') - output = module.execute_command(full_command, extra_environment={'BORG_PASSPHRASE_FD': '4'}) + output = module.execute_command(full_command, environment={'BORG_PASSPHRASE_FD': '4'}) assert output is None @@ -356,7 +355,6 @@ def test_execute_command_without_run_to_completion_returns_process(): full_command = ['foo', 'bar'] process = flexmock() flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, @@ -377,7 +375,6 @@ def test_execute_command_and_capture_output_returns_stdout(): full_command = ['foo', 'bar'] expected_output = '[]' flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('check_output').with_args( full_command, stderr=None, @@ -396,7 +393,6 @@ def test_execute_command_and_capture_output_with_capture_stderr_returns_stderr() full_command = ['foo', 'bar'] expected_output = '[]' flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('check_output').with_args( full_command, stderr=module.subprocess.STDOUT, @@ -416,7 +412,6 @@ def test_execute_command_and_capture_output_returns_output_when_process_error_is expected_output = '[]' err_output = b'[]' flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('check_output').with_args( full_command, stderr=None, @@ -438,7 +433,6 @@ def test_execute_command_and_capture_output_raises_when_command_errors(): full_command = ['foo', 'bar'] expected_output = '[]' flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('check_output').with_args( full_command, stderr=None, @@ -459,7 +453,6 @@ def test_execute_command_and_capture_output_returns_output_with_shell(): full_command = ['foo', 'bar'] expected_output = '[]' flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('check_output').with_args( 'foo bar', stderr=None, @@ -474,22 +467,21 @@ def test_execute_command_and_capture_output_returns_output_with_shell(): assert output == expected_output -def test_execute_command_and_capture_output_returns_output_with_extra_environment(): +def test_execute_command_and_capture_output_returns_output_with_environment(): full_command = ['foo', 'bar'] expected_output = '[]' flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('check_output').with_args( full_command, stderr=None, shell=False, - env={'a': 'b', 'c': 'd'}, + env={'a': 'b'}, cwd=None, close_fds=True, ).and_return(flexmock(decode=lambda: expected_output)).once() output = module.execute_command_and_capture_output( - full_command, shell=False, extra_environment={'c': 'd'} + full_command, shell=False, environment={'a': 'b'} ) assert output == expected_output @@ -499,7 +491,6 @@ def test_execute_command_and_capture_output_returns_output_with_working_director full_command = ['foo', 'bar'] expected_output = '[]' flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('check_output').with_args( full_command, stderr=None, @@ -520,12 +511,11 @@ def test_execute_command_and_capture_output_with_BORG_PASSPHRASE_FD_leaves_file_ full_command = ['foo', 'bar'] expected_output = '[]' flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('check_output').with_args( full_command, stderr=None, shell=False, - env={'a': 'b', 'BORG_PASSPHRASE_FD': '4'}, + env={'BORG_PASSPHRASE_FD': '4'}, cwd=None, close_fds=False, ).and_return(flexmock(decode=lambda: expected_output)).once() @@ -533,7 +523,7 @@ def test_execute_command_and_capture_output_with_BORG_PASSPHRASE_FD_leaves_file_ output = module.execute_command_and_capture_output( full_command, shell=False, - extra_environment={'BORG_PASSPHRASE_FD': '4'}, + environment={'BORG_PASSPHRASE_FD': '4'}, ) assert output == expected_output @@ -543,7 +533,6 @@ def test_execute_command_with_processes_calls_full_command(): full_command = ['foo', 'bar'] processes = (flexmock(),) flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, @@ -566,7 +555,6 @@ def test_execute_command_with_processes_returns_output_with_output_log_level_non full_command = ['foo', 'bar'] processes = (flexmock(),) flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) process = flexmock(stdout=None) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, @@ -591,7 +579,6 @@ def test_execute_command_with_processes_calls_full_command_with_output_file(): processes = (flexmock(),) output_file = flexmock(name='test') flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, @@ -614,7 +601,6 @@ def test_execute_command_with_processes_calls_full_command_without_capturing_out full_command = ['foo', 'bar'] processes = (flexmock(),) flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, @@ -641,7 +627,6 @@ def test_execute_command_with_processes_calls_full_command_with_input_file(): processes = (flexmock(),) input_file = flexmock(name='test') flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=input_file, @@ -664,7 +649,6 @@ def test_execute_command_with_processes_calls_full_command_with_shell(): full_command = ['foo', 'bar'] processes = (flexmock(),) flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( ' '.join(full_command), stdin=None, @@ -683,27 +667,24 @@ def test_execute_command_with_processes_calls_full_command_with_shell(): assert output is None -def test_execute_command_with_processes_calls_full_command_with_extra_environment(): +def test_execute_command_with_processes_calls_full_command_with_environment(): full_command = ['foo', 'bar'] processes = (flexmock(),) flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, stdout=module.subprocess.PIPE, stderr=module.subprocess.STDOUT, shell=False, - env={'a': 'b', 'c': 'd'}, + env={'a': 'b'}, cwd=None, close_fds=True, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') - output = module.execute_command_with_processes( - full_command, processes, extra_environment={'c': 'd'} - ) + output = module.execute_command_with_processes(full_command, processes, environment={'a': 'b'}) assert output is None @@ -712,7 +693,6 @@ def test_execute_command_with_processes_calls_full_command_with_working_director full_command = ['foo', 'bar'] processes = (flexmock(),) flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, @@ -737,14 +717,13 @@ def test_execute_command_with_processes_with_BORG_PASSPHRASE_FD_leaves_file_desc full_command = ['foo', 'bar'] processes = (flexmock(),) flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, stdout=module.subprocess.PIPE, stderr=module.subprocess.STDOUT, shell=False, - env={'a': 'b', 'BORG_PASSPHRASE_FD': '4'}, + env={'BORG_PASSPHRASE_FD': '4'}, cwd=None, close_fds=False, ).and_return(flexmock(stdout=None)).once() @@ -754,7 +733,7 @@ def test_execute_command_with_processes_with_BORG_PASSPHRASE_FD_leaves_file_desc output = module.execute_command_with_processes( full_command, processes, - extra_environment={'BORG_PASSPHRASE_FD': '4'}, + environment={'BORG_PASSPHRASE_FD': '4'}, ) assert output is None @@ -767,7 +746,6 @@ def test_execute_command_with_processes_kills_processes_on_error(): process.should_receive('poll') process.should_receive('kill').once() processes = (process,) - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, From 0e651695030608d06b659bde5f65b1e806d9d7a9 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 17 Feb 2025 14:12:55 -0800 Subject: [PATCH 030/226] Improve clarity of comments and variable names of runtime directory exclude detection logic (#999). --- borgmatic/borg/create.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 924c50e7..692f947d 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -132,9 +132,12 @@ def collect_special_file_paths( used. Skip looking for special files in the given borgmatic runtime directory, as borgmatic creates - its own special files there for database dumps. And if the borgmatic runtime directory is - configured to be excluded from the files Borg backs up, error, because this means Borg won't be - able to consume any database dumps and therefore borgmatic will hang. + its own special files there for database dumps and we don't want those omitted. + + Additionally, if the borgmatic runtime directory is not contained somewhere in the files Borg + plans to backup, that means the user must have excluded the runtime directory (e.g. via + "exclude_patterns" or similar). Therefore, raise, because this means Borg won't be able to + consume any database dumps and therefore borgmatic will hang when it tries to do so. ''' # Omit "--exclude-nodump" from the Borg dry run command, because that flag causes Borg to open # files including any named pipe we've created. @@ -148,25 +151,33 @@ def collect_special_file_paths( borg_exit_codes=config.get('borg_exit_codes'), ) + # These are all the individual files that Borg is planning to backup as determined by the Borg + # create dry run above. paths = tuple( path_line.split(' ', 1)[1] for path_line in paths_output.split('\n') if path_line and path_line.startswith('- ') or path_line.startswith('+ ') ) - skip_paths = {} + + # These are the subset of those files that contain the borgmatic runtime directory. + paths_containing_runtime_directory = {} if os.path.exists(borgmatic_runtime_directory): - skip_paths = { + paths_containing_runtime_directory = { path for path in paths if any_parent_directories(path, (borgmatic_runtime_directory,)) } - if not skip_paths and not dry_run: + # No paths to backup contain the runtime directory, so therefore it must've been excluded. + if not paths_containing_runtime_directory and not dry_run: raise ValueError( f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes or patterns with excludes. Please ensure the runtime directory is not excluded.' ) return tuple( - path for path in paths if special_file(path, working_directory) if path not in skip_paths + path + for path in paths + if special_file(path, working_directory) + if path not in paths_containing_runtime_directory ) From 58aed0892c4a866817e671a4dbcb56e29ad7eec3 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 19 Feb 2025 22:49:14 -0800 Subject: [PATCH 031/226] Initial work on fixing ZFS mount errors (#1001). --- borgmatic/actions/create.py | 7 ++- borgmatic/borg/create.py | 3 +- borgmatic/borg/pattern.py | 21 ++++++++- borgmatic/hooks/data_source/bootstrap.py | 12 ++++- borgmatic/hooks/data_source/btrfs.py | 16 +++++-- borgmatic/hooks/data_source/lvm.py | 37 +++++++++++++--- borgmatic/hooks/data_source/mariadb.py | 3 +- borgmatic/hooks/data_source/mongodb.py | 3 +- borgmatic/hooks/data_source/mysql.py | 3 +- borgmatic/hooks/data_source/postgresql.py | 3 +- borgmatic/hooks/data_source/snapshot.py | 16 ++++--- borgmatic/hooks/data_source/sqlite.py | 3 +- borgmatic/hooks/data_source/zfs.py | 54 ++++++++++++++++++----- 13 files changed, 143 insertions(+), 38 deletions(-) diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index ab7815e4..dede448a 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -36,6 +36,7 @@ def parse_pattern(pattern_line, default_style=borgmatic.borg.pattern.Pattern_sty path, borgmatic.borg.pattern.Pattern_type(pattern_type), borgmatic.borg.pattern.Pattern_style(pattern_style), + source=borgmatic.borg.pattern.Pattern_source.CONFIG, ) @@ -51,7 +52,9 @@ def collect_patterns(config): try: return ( tuple( - borgmatic.borg.pattern.Pattern(source_directory) + borgmatic.borg.pattern.Pattern( + source_directory, source=borgmatic.borg.pattern.Pattern_source.CONFIG + ) for source_directory in config.get('source_directories', ()) ) + tuple( @@ -144,6 +147,7 @@ def expand_patterns(patterns, working_directory=None, skip_paths=None): pattern.type, pattern.style, pattern.device, + pattern.source, ) for expanded_path in expand_directory(pattern.path, working_directory) ) @@ -178,6 +182,7 @@ def device_map_patterns(patterns, working_directory=None): and os.path.exists(full_path) else None ), + source=pattern.source, ) for pattern in patterns for full_path in (os.path.join(working_directory or '', pattern.path),) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 692f947d..b30e42bd 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -167,7 +167,7 @@ def collect_special_file_paths( path for path in paths if any_parent_directories(path, (borgmatic_runtime_directory,)) } - # No paths to backup contain the runtime directory, so therefore it must've been excluded. + # If no paths to backup contain the runtime directory, it must've been excluded. if not paths_containing_runtime_directory and not dry_run: raise ValueError( f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes or patterns with excludes. Please ensure the runtime directory is not excluded.' @@ -336,6 +336,7 @@ def make_base_create_command( special_file_path, borgmatic.borg.pattern.Pattern_type.NO_RECURSE, borgmatic.borg.pattern.Pattern_style.FNMATCH, + source=borgmatic.borg.pattern.Pattern_source.INTERNAL, ) for special_file_path in special_file_paths ), diff --git a/borgmatic/borg/pattern.py b/borgmatic/borg/pattern.py index be57bc0e..dfce7c3b 100644 --- a/borgmatic/borg/pattern.py +++ b/borgmatic/borg/pattern.py @@ -20,12 +20,31 @@ class Pattern_style(enum.Enum): PATH_FULL_MATCH = 'pf' +class Pattern_source(enum.Enum): + ''' + Where the pattern came from within borgmatic. This is important because certain use cases (like + filesystem snapshotting) only want to consider patterns that the user actually put in a + configuration file and not patterns from other sources. + ''' + + # The pattern is from a borgmatic configuration option, e.g. listed in "source_directories". + CONFIG = 'config' + + # The pattern is generated internally within borgmatic, e.g. for special file excludes. + INTERNAL = 'internal' + + # The pattern originates from within a borgmatic hook, e.g. a database hook that adds its dump + # directory. + HOOK = 'hook' + + Pattern = collections.namedtuple( 'Pattern', - ('path', 'type', 'style', 'device'), + ('path', 'type', 'style', 'device', 'source'), defaults=( Pattern_type.ROOT, Pattern_style.NONE, None, + Pattern_source.HOOK, ), ) diff --git a/borgmatic/hooks/data_source/bootstrap.py b/borgmatic/hooks/data_source/bootstrap.py index 67d75421..fe779f0e 100644 --- a/borgmatic/hooks/data_source/bootstrap.py +++ b/borgmatic/hooks/data_source/bootstrap.py @@ -55,9 +55,17 @@ def dump_data_sources( manifest_file, ) - patterns.extend(borgmatic.borg.pattern.Pattern(config_path) for config_path in config_paths) + patterns.extend( + borgmatic.borg.pattern.Pattern( + config_path, source=borgmatic.borg.pattern.Pattern_source.HOOK + ) + for config_path in config_paths + ) patterns.append( - borgmatic.borg.pattern.Pattern(os.path.join(borgmatic_runtime_directory, 'bootstrap')) + borgmatic.borg.pattern.Pattern( + os.path.join(borgmatic_runtime_directory, 'bootstrap'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, + ) ) return [] diff --git a/borgmatic/hooks/data_source/btrfs.py b/borgmatic/hooks/data_source/btrfs.py index 2971aed8..b81a9b05 100644 --- a/borgmatic/hooks/data_source/btrfs.py +++ b/borgmatic/hooks/data_source/btrfs.py @@ -54,7 +54,9 @@ def get_subvolumes(btrfs_command, findmnt_command, patterns=None): between the current Btrfs filesystem and subvolume mount points and the paths of any patterns. The idea is that these pattern paths represent the requested subvolumes to snapshot. - If patterns is None, then return all subvolumes, sorted by path. + Only include subvolumes that contain at least one root pattern sourced from borgmatic + configuration (as opposed to generated elsewhere in borgmatic). But if patterns is None, then + return all subvolumes instead, sorted by path. Return the result as a sequence of matching subvolume mount points. ''' @@ -73,7 +75,12 @@ def get_subvolumes(btrfs_command, findmnt_command, patterns=None): mount_point, candidate_patterns ), ) - if patterns is None or contained_patterns + if patterns is None + or any( + pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT + and pattern.source == borgmatic.borg.pattern.Pattern_source.CONFIG + for pattern in contained_patterns + ) ) return tuple(sorted(subvolumes, key=lambda subvolume: subvolume.path)) @@ -118,6 +125,7 @@ def make_snapshot_exclude_pattern(subvolume_path): # pragma: no cover snapshot_directory, subvolume_path.lstrip(os.path.sep), snapshot_directory, + source=borgmatic.borg.pattern.Pattern_source.HOOK, ), borgmatic.borg.pattern.Pattern_type.NO_RECURSE, borgmatic.borg.pattern.Pattern_style.FNMATCH, @@ -153,6 +161,7 @@ def make_borg_snapshot_pattern(subvolume_path, pattern): pattern.type, pattern.style, pattern.device, + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) @@ -198,7 +207,8 @@ def dump_data_sources( dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' logger.info(f'Snapshotting Btrfs subvolumes{dry_run_label}') - # Based on the configured patterns, determine Btrfs subvolumes to backup. + # Based on the configured patterns, determine Btrfs subvolumes to backup. Only consider those + # patterns that came from actual user configuration (as opposed to, say, other hooks). btrfs_command = hook_config.get('btrfs_command', 'btrfs') findmnt_command = hook_config.get('findmnt_command', 'findmnt') subvolumes = get_subvolumes(btrfs_command, findmnt_command, patterns) diff --git a/borgmatic/hooks/data_source/lvm.py b/borgmatic/hooks/data_source/lvm.py index 8436d0f6..8743f3e1 100644 --- a/borgmatic/hooks/data_source/lvm.py +++ b/borgmatic/hooks/data_source/lvm.py @@ -1,5 +1,6 @@ import collections import glob +import hashlib import json import logging import os @@ -33,7 +34,9 @@ def get_logical_volumes(lsblk_command, patterns=None): between the current LVM logical volume mount points and the paths of any patterns. The idea is that these pattern paths represent the requested logical volumes to snapshot. - If patterns is None, include all logical volume mounts points, not just those in patterns. + Only include logical volumes that contain at least one root pattern sourced from borgmatic + configuration (as opposed to generated elsewhere in borgmatic). But if patterns is None, include + all logical volume mounts points instead, not just those in patterns. Return the result as a sequence of Logical_volume instances. ''' @@ -72,7 +75,12 @@ def get_logical_volumes(lsblk_command, patterns=None): device['mountpoint'], candidate_patterns ), ) - if not patterns or contained_patterns + if not patterns + or any( + pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT + and pattern.source == borgmatic.borg.pattern.Pattern_source.CONFIG + for pattern in contained_patterns + ) ) except KeyError as error: raise ValueError(f'Invalid {lsblk_command} output: Missing key "{error}"') @@ -124,10 +132,14 @@ def mount_snapshot(mount_command, snapshot_device, snapshot_mount_path): # prag ) -def make_borg_snapshot_pattern(pattern, normalized_runtime_directory): +MOUNT_POINT_HASH_LENGTH = 10 + + +def make_borg_snapshot_pattern(pattern, logical_volume, normalized_runtime_directory): ''' - Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance, return a new Pattern with its - path rewritten to be in a snapshot directory based on the given runtime directory. + Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance and a Logical_volume + contianing it, return a new Pattern with its path rewritten to be in a snapshot directory based + on both the given runtime directory and the given Logical_volume's mount point. Move any initial caret in a regular expression pattern path to the beginning, so as not to break the regular expression. @@ -142,6 +154,11 @@ def make_borg_snapshot_pattern(pattern, normalized_runtime_directory): rewritten_path = initial_caret + os.path.join( normalized_runtime_directory, 'lvm_snapshots', + # Including this hash prevents conflicts between snapshot patterns for different logical + # volumes. For instance, without this, snapshotting a logical volume at /var and another at + # /var/spool would result in overlapping snapshot patterns and therefore colliding mount + # attempts. + hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest(MOUNT_POINT_HASH_LENGTH), '.', # Borg 1.4+ "slashdot" hack. # Included so that the source directory ends up in the Borg archive at its "original" path. pattern.path.lstrip('^').lstrip(os.path.sep), @@ -152,6 +169,7 @@ def make_borg_snapshot_pattern(pattern, normalized_runtime_directory): pattern.type, pattern.style, pattern.device, + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) @@ -180,7 +198,8 @@ def dump_data_sources( dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' logger.info(f'Snapshotting LVM logical volumes{dry_run_label}') - # List logical volumes to get their mount points. + # List logical volumes to get their mount points, but only consider those patterns that came + # from actual user configuration (as opposed to, say, other hooks). lsblk_command = hook_config.get('lsblk_command', 'lsblk') requested_logical_volumes = get_logical_volumes(lsblk_command, patterns) @@ -218,6 +237,7 @@ def dump_data_sources( snapshot_mount_path = os.path.join( normalized_runtime_directory, 'lvm_snapshots', + hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest(MOUNT_POINT_HASH_LENGTH), logical_volume.mount_point.lstrip(os.path.sep), ) @@ -233,7 +253,9 @@ def dump_data_sources( ) for pattern in logical_volume.contained_patterns: - snapshot_pattern = make_borg_snapshot_pattern(pattern, normalized_runtime_directory) + snapshot_pattern = make_borg_snapshot_pattern( + pattern, logical_volume, normalized_runtime_directory + ) # Attempt to update the pattern in place, since pattern order matters to Borg. try: @@ -337,6 +359,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d os.path.normpath(borgmatic_runtime_directory), ), 'lvm_snapshots', + '*', ) logger.debug(f'Looking for snapshots to remove in {snapshots_glob}{dry_run_label}') umount_command = hook_config.get('umount_command', 'umount') diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index 47d1dfea..0be03941 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -215,7 +215,8 @@ def dump_data_sources( if not dry_run: patterns.append( borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'mariadb_databases') + os.path.join(borgmatic_runtime_directory, 'mariadb_databases'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) ) diff --git a/borgmatic/hooks/data_source/mongodb.py b/borgmatic/hooks/data_source/mongodb.py index 2148f009..59811101 100644 --- a/borgmatic/hooks/data_source/mongodb.py +++ b/borgmatic/hooks/data_source/mongodb.py @@ -81,7 +81,8 @@ def dump_data_sources( if not dry_run: patterns.append( borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'mongodb_databases') + os.path.join(borgmatic_runtime_directory, 'mongodb_databases'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) ) diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index aa5a954e..48594e54 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -214,7 +214,8 @@ def dump_data_sources( if not dry_run: patterns.append( borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'mysql_databases') + os.path.join(borgmatic_runtime_directory, 'mysql_databases'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) ) diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index a9eafea0..8eb7c62c 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -241,7 +241,8 @@ def dump_data_sources( if not dry_run: patterns.append( borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'postgresql_databases') + os.path.join(borgmatic_runtime_directory, 'postgresql_databases'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) ) diff --git a/borgmatic/hooks/data_source/snapshot.py b/borgmatic/hooks/data_source/snapshot.py index 2ee08a8f..b13f1aa3 100644 --- a/borgmatic/hooks/data_source/snapshot.py +++ b/borgmatic/hooks/data_source/snapshot.py @@ -1,3 +1,5 @@ +import logging + import pathlib IS_A_HOOK = False @@ -11,10 +13,10 @@ def get_contained_patterns(parent_directory, candidate_patterns): paths, but there's a parent directory (logical volume, dataset, subvolume, etc.) at /var, then /var is what we want to snapshot. - For this to work, a candidate pattern path can't have any globs or other non-literal characters - in the initial portion of the path that matches the parent directory. For instance, a parent - directory of /var would match a candidate pattern path of /var/log/*/data, but not a pattern - path like /v*/log/*/data. + For this function to work, a candidate pattern path can't have any globs or other non-literal + characters in the initial portion of the path that matches the parent directory. For instance, a + parent directory of /var would match a candidate pattern path of /var/log/*/data, but not a + pattern path like /v*/log/*/data. The one exception is that if a regular expression pattern path starts with "^", that will get stripped off for purposes of matching against a parent directory. @@ -31,8 +33,10 @@ def get_contained_patterns(parent_directory, candidate_patterns): candidate for candidate in candidate_patterns for candidate_path in (pathlib.PurePath(candidate.path.lstrip('^')),) - if pathlib.PurePath(parent_directory) == candidate_path - or pathlib.PurePath(parent_directory) in candidate_path.parents + if ( + pathlib.PurePath(parent_directory) == candidate_path + or pathlib.PurePath(parent_directory) in candidate_path.parents + ) ) candidate_patterns -= set(contained_patterns) diff --git a/borgmatic/hooks/data_source/sqlite.py b/borgmatic/hooks/data_source/sqlite.py index 4d5ce9a0..b36c0ccb 100644 --- a/borgmatic/hooks/data_source/sqlite.py +++ b/borgmatic/hooks/data_source/sqlite.py @@ -90,7 +90,8 @@ def dump_data_sources( if not dry_run: patterns.append( borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'sqlite_databases') + os.path.join(borgmatic_runtime_directory, 'sqlite_databases'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) ) diff --git a/borgmatic/hooks/data_source/zfs.py b/borgmatic/hooks/data_source/zfs.py index 82a71e7a..105eb215 100644 --- a/borgmatic/hooks/data_source/zfs.py +++ b/borgmatic/hooks/data_source/zfs.py @@ -1,5 +1,6 @@ import collections import glob +import hashlib import logging import os import shutil @@ -38,6 +39,9 @@ def get_datasets_to_backup(zfs_command, patterns): pattern paths represent the requested datasets to snapshot. But also include any datasets tagged with a borgmatic-specific user property, whether or not they appear in the patterns. + Only include datasets that contain at least one root pattern sourced from borgmatic + configuration (as opposed to generated elsewhere in borgmatic). + Return the result as a sequence of Dataset instances, sorted by mount point. ''' list_output = borgmatic.execute.execute_command_and_capture_output( @@ -48,7 +52,7 @@ def get_datasets_to_backup(zfs_command, patterns): '-t', 'filesystem', '-o', - f'name,mountpoint,{BORGMATIC_USER_PROPERTY}', + f'name,mountpoint,canmount,{BORGMATIC_USER_PROPERTY}', ) ) @@ -60,7 +64,10 @@ def get_datasets_to_backup(zfs_command, patterns): ( Dataset(dataset_name, mount_point, (user_property_value == 'auto'), ()) for line in list_output.splitlines() - for (dataset_name, mount_point, user_property_value) in (line.rstrip().split('\t'),) + for (dataset_name, mount_point, can_mount, user_property_value) in (line.rstrip().split('\t'),) + # Skip datasets that are marked "canmount=off", because mounting their snapshots will + # result in completely empty mount points—thereby preventing us from backing them up. + if can_mount == 'on' ), key=lambda dataset: dataset.mount_point, reverse=True, @@ -83,7 +90,12 @@ def get_datasets_to_backup(zfs_command, patterns): for contained_patterns in ( ( ( - (borgmatic.borg.pattern.Pattern(dataset.mount_point),) + ( + borgmatic.borg.pattern.Pattern( + dataset.mount_point, + source=borgmatic.borg.pattern.Pattern_source.HOOK, + ), + ) if dataset.auto_backup else () ) @@ -92,7 +104,11 @@ def get_datasets_to_backup(zfs_command, patterns): ) ), ) - if contained_patterns + if any( + pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT + and pattern.source == borgmatic.borg.pattern.Pattern_source.CONFIG + for pattern in contained_patterns + ) ), key=lambda dataset: dataset.mount_point, ) @@ -155,10 +171,14 @@ def mount_snapshot(mount_command, full_snapshot_name, snapshot_mount_path): # p ) -def make_borg_snapshot_pattern(pattern, normalized_runtime_directory): +MOUNT_POINT_HASH_LENGTH = 10 + + +def make_borg_snapshot_pattern(pattern, dataset, normalized_runtime_directory): ''' - Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance, return a new Pattern with its - path rewritten to be in a snapshot directory based on the given runtime directory. + Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance and the Dataset containing it, + return a new Pattern with its path rewritten to be in a snapshot directory based on both the + given runtime directory and the given Dataset's mount point. Move any initial caret in a regular expression pattern path to the beginning, so as not to break the regular expression. @@ -173,6 +193,10 @@ def make_borg_snapshot_pattern(pattern, normalized_runtime_directory): rewritten_path = initial_caret + os.path.join( normalized_runtime_directory, 'zfs_snapshots', + # Including this hash prevents conflicts between snapshot patterns for different datasets. + # For instance, without this, snapshotting a dataset at /var and another at /var/spool would + # result in overlapping snapshot patterns and therefore colliding mount attempts. + hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest(MOUNT_POINT_HASH_LENGTH), '.', # Borg 1.4+ "slashdot" hack. # Included so that the source directory ends up in the Borg archive at its "original" path. pattern.path.lstrip('^').lstrip(os.path.sep), @@ -183,6 +207,7 @@ def make_borg_snapshot_pattern(pattern, normalized_runtime_directory): pattern.type, pattern.style, pattern.device, + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) @@ -209,7 +234,8 @@ def dump_data_sources( dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' logger.info(f'Snapshotting ZFS datasets{dry_run_label}') - # List ZFS datasets to get their mount points. + # List ZFS datasets to get their mount points, but only consider those patterns that came from + # actual user configuration (as opposed to, say, other hooks). zfs_command = hook_config.get('zfs_command', 'zfs') requested_datasets = get_datasets_to_backup(zfs_command, patterns) @@ -234,6 +260,7 @@ def dump_data_sources( snapshot_mount_path = os.path.join( normalized_runtime_directory, 'zfs_snapshots', + hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest(MOUNT_POINT_HASH_LENGTH), dataset.mount_point.lstrip(os.path.sep), ) @@ -249,7 +276,9 @@ def dump_data_sources( ) for pattern in dataset.contained_patterns: - snapshot_pattern = make_borg_snapshot_pattern(pattern, normalized_runtime_directory) + snapshot_pattern = make_borg_snapshot_pattern( + pattern, dataset, normalized_runtime_directory + ) # Attempt to update the pattern in place, since pattern order matters to Borg. try: @@ -334,6 +363,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d os.path.normpath(borgmatic_runtime_directory), ), 'zfs_snapshots', + '*', ) logger.debug(f'Looking for snapshots to remove in {snapshots_glob}{dry_run_label}') umount_command = hook_config.get('umount_command', 'umount') @@ -367,13 +397,13 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d unmount_snapshot(umount_command, snapshot_mount_path) except FileNotFoundError: logger.debug(f'Could not find "{umount_command}" command') - return + continue except subprocess.CalledProcessError as error: logger.debug(error) - return + continue if not dry_run: - shutil.rmtree(snapshots_directory) + shutil.rmtree(snapshots_directory, ignore_errors=True) # Destroy snapshots. full_snapshot_names = get_all_snapshots(zfs_command) From 3655e8784ae99a1ead6e451af3344f502e68733f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Feb 2025 13:25:09 -0800 Subject: [PATCH 032/226] Add NEWS items for filesystem hook fixes/changes (#1001). --- NEWS | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index b8370afa..893e8047 100644 --- a/NEWS +++ b/NEWS @@ -1,9 +1,15 @@ 1.9.11.dev0 - * #996: Fix the "create" action to omit the repository label prefix from Borg's output when - databases are enabled. * #795: Add credential loading from file, KeePassXC, and Docker/Podman secrets. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/ + * #996: Fix the "create" action to omit the repository label prefix from Borg's output when + databases are enabled. + * #1001: For the ZFS, Btrfs, and LVM hooks, only make snapshots for patterns that come from + a borgmatic configuration option (e.g. "source_directories")—not from other hooks within + borgmatic. + * #1001: Fix a ZFS/LVM error due to colliding snapshot mount points for nested datasets or logical + volumes. + * #1001: Don't try to snapshot ZFS datasets that have the "canmount=off" property. * Fix another error in the Btrfs hook when a subvolume mounted at "/" is configured in borgmatic's source directories. From e69cce7e510bfa41cb9fb14dd269d51ca5421f93 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Feb 2025 14:04:23 -0800 Subject: [PATCH 033/226] Document ZFS snapshotting exclusion of "canmount=off" datasets (#1001). --- docs/how-to/snapshot-your-filesystems.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/how-to/snapshot-your-filesystems.md b/docs/how-to/snapshot-your-filesystems.md index a9f2d267..36078ead 100644 --- a/docs/how-to/snapshot-your-filesystems.md +++ b/docs/how-to/snapshot-your-filesystems.md @@ -65,6 +65,11 @@ If you have multiple borgmatic configuration files with ZFS enabled, and you'd like particular datasets to be backed up only for particular configuration files, use the `source_directories` option instead of the user property. +New in version 1.9.11 borgmatic +won't snapshot datasets with the `canmount=off` property, which is often set on +datasets that only serve as a container for other datasets. Use `zfs get +canmount datasetname` to see the `canmount` value for a dataset. + During a backup, borgmatic automatically snapshots these discovered datasets (non-recursively), temporarily mounts the snapshots within its [runtime directory](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#runtime-directory), From 0210bf76bc9a603392f54b15d6ea868a20b53390 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Feb 2025 22:58:05 -0800 Subject: [PATCH 034/226] Fix ZFS and Btrfs tests (#1001). --- NEWS | 2 +- borgmatic/hooks/data_source/lvm.py | 4 +- borgmatic/hooks/data_source/zfs.py | 15 +- tests/end-to-end/commands/fake_zfs.py | 2 + tests/unit/hooks/data_source/test_btrfs.py | 71 ++++- tests/unit/hooks/data_source/test_zfs.py | 308 ++++++++++++++++----- 6 files changed, 326 insertions(+), 76 deletions(-) diff --git a/NEWS b/NEWS index 893e8047..04f07bcf 100644 --- a/NEWS +++ b/NEWS @@ -4,7 +4,7 @@ https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/ * #996: Fix the "create" action to omit the repository label prefix from Borg's output when databases are enabled. - * #1001: For the ZFS, Btrfs, and LVM hooks, only make snapshots for patterns that come from + * #1001: For the ZFS, Btrfs, and LVM hooks, only make snapshots for root patterns that come from a borgmatic configuration option (e.g. "source_directories")—not from other hooks within borgmatic. * #1001: Fix a ZFS/LVM error due to colliding snapshot mount points for nested datasets or logical diff --git a/borgmatic/hooks/data_source/lvm.py b/borgmatic/hooks/data_source/lvm.py index 8743f3e1..1d2e4b42 100644 --- a/borgmatic/hooks/data_source/lvm.py +++ b/borgmatic/hooks/data_source/lvm.py @@ -237,7 +237,9 @@ def dump_data_sources( snapshot_mount_path = os.path.join( normalized_runtime_directory, 'lvm_snapshots', - hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest(MOUNT_POINT_HASH_LENGTH), + hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest( + MOUNT_POINT_HASH_LENGTH + ), logical_volume.mount_point.lstrip(os.path.sep), ) diff --git a/borgmatic/hooks/data_source/zfs.py b/borgmatic/hooks/data_source/zfs.py index 105eb215..61dfeb22 100644 --- a/borgmatic/hooks/data_source/zfs.py +++ b/borgmatic/hooks/data_source/zfs.py @@ -64,7 +64,9 @@ def get_datasets_to_backup(zfs_command, patterns): ( Dataset(dataset_name, mount_point, (user_property_value == 'auto'), ()) for line in list_output.splitlines() - for (dataset_name, mount_point, can_mount, user_property_value) in (line.rstrip().split('\t'),) + for (dataset_name, mount_point, can_mount, user_property_value) in ( + line.rstrip().split('\t'), + ) # Skip datasets that are marked "canmount=off", because mounting their snapshots will # result in completely empty mount points—thereby preventing us from backing them up. if can_mount == 'on' @@ -104,7 +106,8 @@ def get_datasets_to_backup(zfs_command, patterns): ) ), ) - if any( + if dataset.auto_backup + or any( pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT and pattern.source == borgmatic.borg.pattern.Pattern_source.CONFIG for pattern in contained_patterns @@ -260,7 +263,9 @@ def dump_data_sources( snapshot_mount_path = os.path.join( normalized_runtime_directory, 'zfs_snapshots', - hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest(MOUNT_POINT_HASH_LENGTH), + hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest( + MOUNT_POINT_HASH_LENGTH + ), dataset.mount_point.lstrip(os.path.sep), ) @@ -369,6 +374,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d umount_command = hook_config.get('umount_command', 'umount') for snapshots_directory in glob.glob(snapshots_glob): + print('*** snapshots_glob:', snapshots_glob, 'snapshots_directory:', snapshots_directory) if not os.path.isdir(snapshots_directory): continue @@ -376,6 +382,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d # child datasets before the shorter mount point paths of parent datasets. for mount_point in reversed(dataset_mount_points): snapshot_mount_path = os.path.join(snapshots_directory, mount_point.lstrip(os.path.sep)) + print('*** snapshot_mount_path:', snapshot_mount_path) if not os.path.isdir(snapshot_mount_path): continue @@ -397,7 +404,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d unmount_snapshot(umount_command, snapshot_mount_path) except FileNotFoundError: logger.debug(f'Could not find "{umount_command}" command') - continue + return except subprocess.CalledProcessError as error: logger.debug(error) continue diff --git a/tests/end-to-end/commands/fake_zfs.py b/tests/end-to-end/commands/fake_zfs.py index 61f8ae5e..3c13f4d5 100644 --- a/tests/end-to-end/commands/fake_zfs.py +++ b/tests/end-to-end/commands/fake_zfs.py @@ -27,6 +27,7 @@ BUILTIN_DATASETS = ( 'used': '256K', 'avail': '23.7M', 'refer': '25K', + 'canmount': 'on', 'mountpoint': '/pool', }, { @@ -34,6 +35,7 @@ BUILTIN_DATASETS = ( 'used': '256K', 'avail': '23.7M', 'refer': '25K', + 'canmount': 'on', 'mountpoint': '/pool/dataset', }, ) diff --git a/tests/unit/hooks/data_source/test_btrfs.py b/tests/unit/hooks/data_source/test_btrfs.py index dde7b05b..65c4ccbe 100644 --- a/tests/unit/hooks/data_source/test_btrfs.py +++ b/tests/unit/hooks/data_source/test_btrfs.py @@ -52,9 +52,14 @@ def test_get_subvolume_mount_points_with_findmnt_json_missing_filesystems_errors def test_get_subvolumes_collects_subvolumes_matching_patterns(): flexmock(module).should_receive('get_subvolume_mount_points').and_return(('/mnt1', '/mnt2')) + contained_pattern = Pattern( + '/mnt1', + type=module.borgmatic.borg.pattern.Pattern_type.ROOT, + source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + ) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' - ).with_args('/mnt1', object).and_return((Pattern('/mnt1'),)) + ).with_args('/mnt1', object).and_return((contained_pattern,)) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' ).with_args('/mnt2', object).and_return(()) @@ -66,7 +71,69 @@ def test_get_subvolumes_collects_subvolumes_matching_patterns(): Pattern('/mnt1'), Pattern('/mnt3'), ], - ) == (module.Subvolume('/mnt1', contained_patterns=(Pattern('/mnt1'),)),) + ) == (module.Subvolume('/mnt1', contained_patterns=(contained_pattern,)),) + + +def test_get_subvolumes_skips_non_root_patterns(): + flexmock(module).should_receive('get_subvolume_mount_points').and_return(('/mnt1', '/mnt2')) + + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/mnt1', object).and_return( + ( + Pattern( + '/mnt1', + type=module.borgmatic.borg.pattern.Pattern_type.EXCLUDE, + source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + ), + ) + ) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/mnt2', object).and_return(()) + + assert ( + module.get_subvolumes( + 'btrfs', + 'findmnt', + patterns=[ + Pattern('/mnt1'), + Pattern('/mnt3'), + ], + ) + == () + ) + + +def test_get_subvolumes_skips_non_config_patterns(): + flexmock(module).should_receive('get_subvolume_mount_points').and_return(('/mnt1', '/mnt2')) + + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/mnt1', object).and_return( + ( + Pattern( + '/mnt1', + type=module.borgmatic.borg.pattern.Pattern_type.ROOT, + source=module.borgmatic.borg.pattern.Pattern_source.HOOK, + ), + ) + ) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/mnt2', object).and_return(()) + + assert ( + module.get_subvolumes( + 'btrfs', + 'findmnt', + patterns=[ + Pattern('/mnt1'), + Pattern('/mnt3'), + ], + ) + == () + ) def test_get_subvolumes_without_patterns_collects_all_subvolumes(): diff --git a/tests/unit/hooks/data_source/test_zfs.py b/tests/unit/hooks/data_source/test_zfs.py index 85782bdb..7bc07122 100644 --- a/tests/unit/hooks/data_source/test_zfs.py +++ b/tests/unit/hooks/data_source/test_zfs.py @@ -11,11 +11,19 @@ def test_get_datasets_to_backup_filters_datasets_by_patterns(): flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).and_return( - 'dataset\t/dataset\t-\nother\t/other\t-', + 'dataset\t/dataset\ton\t-\nother\t/other\ton\t-', ) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' - ).with_args('/dataset', object).and_return((Pattern('/dataset'),)) + ).with_args('/dataset', object).and_return( + ( + Pattern( + '/dataset', + module.borgmatic.borg.pattern.Pattern_type.ROOT, + source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + ), + ) + ) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' ).with_args('/other', object).and_return(()) @@ -23,24 +31,128 @@ def test_get_datasets_to_backup_filters_datasets_by_patterns(): assert module.get_datasets_to_backup( 'zfs', patterns=( - Pattern('/foo'), - Pattern('/dataset'), - Pattern('/bar'), + Pattern( + '/foo', + module.borgmatic.borg.pattern.Pattern_type.ROOT, + source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + ), + Pattern( + '/dataset', + module.borgmatic.borg.pattern.Pattern_type.ROOT, + source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + ), + Pattern( + '/bar', + module.borgmatic.borg.pattern.Pattern_type.ROOT, + source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + ), ), ) == ( module.Dataset( name='dataset', mount_point='/dataset', - contained_patterns=(Pattern('/dataset'),), + contained_patterns=( + Pattern( + '/dataset', + module.borgmatic.borg.pattern.Pattern_type.ROOT, + source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + ), + ), ), ) +def test_get_datasets_to_backup_skips_non_root_patterns(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( + 'dataset\t/dataset\ton\t-\nother\t/other\ton\t-', + ) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/dataset', object).and_return( + ( + Pattern( + '/dataset', + module.borgmatic.borg.pattern.Pattern_type.EXCLUDE, + source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + ), + ) + ) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/other', object).and_return(()) + + assert module.get_datasets_to_backup( + 'zfs', + patterns=( + Pattern( + '/foo', + module.borgmatic.borg.pattern.Pattern_type.ROOT, + source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + ), + Pattern( + '/dataset', + module.borgmatic.borg.pattern.Pattern_type.EXCLUDE, + source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + ), + Pattern( + '/bar', + module.borgmatic.borg.pattern.Pattern_type.ROOT, + source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + ), + ), + ) == () + + +def test_get_datasets_to_backup_skips_non_config_patterns(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( + 'dataset\t/dataset\ton\t-\nother\t/other\ton\t-', + ) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/dataset', object).and_return( + ( + Pattern( + '/dataset', + module.borgmatic.borg.pattern.Pattern_type.ROOT, + source=module.borgmatic.borg.pattern.Pattern_source.HOOK, + ), + ) + ) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/other', object).and_return(()) + + assert module.get_datasets_to_backup( + 'zfs', + patterns=( + Pattern( + '/foo', + module.borgmatic.borg.pattern.Pattern_type.ROOT, + source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + ), + Pattern( + '/dataset', + module.borgmatic.borg.pattern.Pattern_type.ROOT, + source=module.borgmatic.borg.pattern.Pattern_source.HOOK, + ), + Pattern( + '/bar', + module.borgmatic.borg.pattern.Pattern_type.ROOT, + source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + ), + ), + ) == () + + def test_get_datasets_to_backup_filters_datasets_by_user_property(): flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).and_return( - 'dataset\t/dataset\tauto\nother\t/other\t-', + 'dataset\t/dataset\ton\tauto\nother\t/other\ton\t-', ) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' @@ -49,16 +161,47 @@ def test_get_datasets_to_backup_filters_datasets_by_user_property(): 'get_contained_patterns' ).with_args('/other', object).and_return(()) - assert module.get_datasets_to_backup('zfs', patterns=(Pattern('/foo'), Pattern('/bar'))) == ( + assert module.get_datasets_to_backup( + 'zfs', + patterns=(Pattern('/foo'), Pattern('/bar')), + ) == ( module.Dataset( name='dataset', mount_point='/dataset', auto_backup=True, - contained_patterns=(Pattern('/dataset'),), + contained_patterns=( + Pattern('/dataset', source=module.borgmatic.borg.pattern.Pattern_source.HOOK), + ), ), ) +def test_get_datasets_to_backup_filters_datasets_by_canmount_property(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( + 'dataset\t/dataset\toff\t-\nother\t/other\ton\t-', + ) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/dataset', object).and_return((Pattern('/dataset'),)) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/other', object).and_return(()) + + assert ( + module.get_datasets_to_backup( + 'zfs', + patterns=( + Pattern('/foo'), + Pattern('/dataset'), + Pattern('/bar'), + ), + ) + == () + ) + + def test_get_datasets_to_backup_with_invalid_list_output_raises(): flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' @@ -94,13 +237,13 @@ def test_get_all_dataset_mount_points_does_not_filter_datasets(): ( ( Pattern('/foo/bar/baz'), - Pattern('/run/borgmatic/zfs_snapshots/./foo/bar/baz'), + Pattern('/run/borgmatic/zfs_snapshots/b33f/./foo/bar/baz'), ), - (Pattern('/foo/bar'), Pattern('/run/borgmatic/zfs_snapshots/./foo/bar')), + (Pattern('/foo/bar'), Pattern('/run/borgmatic/zfs_snapshots/b33f/./foo/bar')), ( Pattern('^/foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION), Pattern( - '^/run/borgmatic/zfs_snapshots/./foo/bar', + '^/run/borgmatic/zfs_snapshots/b33f/./foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION, ), @@ -108,46 +251,55 @@ def test_get_all_dataset_mount_points_does_not_filter_datasets(): ( Pattern('/foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION), Pattern( - '/run/borgmatic/zfs_snapshots/./foo/bar', + '/run/borgmatic/zfs_snapshots/b33f/./foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION, ), ), - (Pattern('/foo'), Pattern('/run/borgmatic/zfs_snapshots/./foo')), - (Pattern('/'), Pattern('/run/borgmatic/zfs_snapshots/./')), + (Pattern('/foo'), Pattern('/run/borgmatic/zfs_snapshots/b33f/./foo')), + (Pattern('/'), Pattern('/run/borgmatic/zfs_snapshots/b33f/./')), ), ) def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path( pattern, expected_pattern ): - assert module.make_borg_snapshot_pattern(pattern, '/run/borgmatic') == expected_pattern + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) + + assert ( + module.make_borg_snapshot_pattern( + pattern, flexmock(mount_point='/something'), '/run/borgmatic' + ) + == expected_pattern + ) def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns(): - flexmock(module).should_receive('get_datasets_to_backup').and_return( - ( - flexmock( - name='dataset', - mount_point='/mnt/dataset', - contained_patterns=(Pattern('/mnt/dataset/subdir'),), - ) - ) + dataset = flexmock( + name='dataset', + mount_point='/mnt/dataset', + contained_patterns=(Pattern('/mnt/dataset/subdir'),), ) + flexmock(module).should_receive('get_datasets_to_backup').and_return((dataset,)) flexmock(module.os).should_receive('getpid').and_return(1234) full_snapshot_name = 'dataset@borgmatic-1234' flexmock(module).should_receive('snapshot_dataset').with_args( 'zfs', full_snapshot_name, ).once() - snapshot_mount_path = '/run/borgmatic/zfs_snapshots/./mnt/dataset' + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) + snapshot_mount_path = '/run/borgmatic/zfs_snapshots/b33f/./mnt/dataset' flexmock(module).should_receive('mount_snapshot').with_args( 'mount', full_snapshot_name, module.os.path.normpath(snapshot_mount_path), ).once() flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/dataset/subdir'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/zfs_snapshots/./mnt/dataset/subdir')) + Pattern('/mnt/dataset/subdir'), dataset, '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/zfs_snapshots/b33f/./mnt/dataset/subdir')) patterns = [Pattern('/mnt/dataset/subdir')] assert ( @@ -188,30 +340,30 @@ def test_dump_data_sources_with_no_datasets_skips_snapshots(): def test_dump_data_sources_uses_custom_commands(): - flexmock(module).should_receive('get_datasets_to_backup').and_return( - ( - flexmock( - name='dataset', - mount_point='/mnt/dataset', - contained_patterns=(Pattern('/mnt/dataset/subdir'),), - ) - ) + dataset = flexmock( + name='dataset', + mount_point='/mnt/dataset', + contained_patterns=(Pattern('/mnt/dataset/subdir'),), ) + flexmock(module).should_receive('get_datasets_to_backup').and_return((dataset,)) flexmock(module.os).should_receive('getpid').and_return(1234) full_snapshot_name = 'dataset@borgmatic-1234' flexmock(module).should_receive('snapshot_dataset').with_args( '/usr/local/bin/zfs', full_snapshot_name, ).once() - snapshot_mount_path = '/run/borgmatic/zfs_snapshots/./mnt/dataset' + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) + snapshot_mount_path = '/run/borgmatic/zfs_snapshots/b33f/./mnt/dataset' flexmock(module).should_receive('mount_snapshot').with_args( '/usr/local/bin/mount', full_snapshot_name, module.os.path.normpath(snapshot_mount_path), ).once() flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/dataset/subdir'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/zfs_snapshots/./mnt/dataset/subdir')) + Pattern('/mnt/dataset/subdir'), dataset, '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/zfs_snapshots/b33f/./mnt/dataset/subdir')) patterns = [Pattern('/mnt/dataset/subdir')] hook_config = { 'zfs_command': '/usr/local/bin/zfs', @@ -261,30 +413,30 @@ def test_dump_data_sources_with_dry_run_skips_commands_and_does_not_touch_patter def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained_patterns(): - flexmock(module).should_receive('get_datasets_to_backup').and_return( - ( - flexmock( - name='dataset', - mount_point='/mnt/dataset', - contained_patterns=(Pattern('/mnt/dataset/subdir'),), - ) - ) + dataset = flexmock( + name='dataset', + mount_point='/mnt/dataset', + contained_patterns=(Pattern('/mnt/dataset/subdir'),), ) + flexmock(module).should_receive('get_datasets_to_backup').and_return((dataset,)) flexmock(module.os).should_receive('getpid').and_return(1234) full_snapshot_name = 'dataset@borgmatic-1234' flexmock(module).should_receive('snapshot_dataset').with_args( 'zfs', full_snapshot_name, ).once() - snapshot_mount_path = '/run/borgmatic/zfs_snapshots/./mnt/dataset' + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) + snapshot_mount_path = '/run/borgmatic/zfs_snapshots/b33f/./mnt/dataset' flexmock(module).should_receive('mount_snapshot').with_args( 'mount', full_snapshot_name, module.os.path.normpath(snapshot_mount_path), ).once() flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/dataset/subdir'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/zfs_snapshots/./mnt/dataset/subdir')) + Pattern('/mnt/dataset/subdir'), dataset, '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/zfs_snapshots/b33f/./mnt/dataset/subdir')) patterns = [Pattern('/hmm')] assert ( @@ -317,11 +469,13 @@ def test_remove_data_source_dumps_unmounts_and_destroys_snapshots(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( - 'umount', '/run/borgmatic/zfs_snapshots/mnt/dataset' + 'umount', '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' ).once() flexmock(module).should_receive('get_all_snapshots').and_return( ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid') @@ -343,11 +497,13 @@ def test_remove_data_source_dumps_use_custom_commands(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( - '/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/mnt/dataset' + '/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' ).once() flexmock(module).should_receive('get_all_snapshots').and_return( ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid') @@ -416,11 +572,13 @@ def test_remove_data_source_dumps_bails_for_missing_umount_command(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( - '/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/mnt/dataset' + '/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' ).and_raise(FileNotFoundError) flexmock(module).should_receive('get_all_snapshots').never() flexmock(module).should_receive('destroy_snapshot').never() @@ -434,19 +592,25 @@ def test_remove_data_source_dumps_bails_for_missing_umount_command(): ) -def test_remove_data_source_dumps_bails_for_umount_command_error(): +def test_remove_data_source_dumps_swallows_umount_command_error(): flexmock(module).should_receive('get_all_dataset_mount_points').and_return(('/mnt/dataset',)) flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( - '/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/mnt/dataset' + '/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' ).and_raise(module.subprocess.CalledProcessError(1, 'wtf')) - flexmock(module).should_receive('get_all_snapshots').never() - flexmock(module).should_receive('destroy_snapshot').never() + flexmock(module).should_receive('get_all_snapshots').and_return( + ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid') + ) + flexmock(module).should_receive('destroy_snapshot').with_args( + '/usr/local/bin/zfs', 'dataset@borgmatic-1234' + ).once() hook_config = {'zfs_command': '/usr/local/bin/zfs', 'umount_command': '/usr/local/bin/umount'} module.remove_data_source_dumps( @@ -462,7 +626,9 @@ def test_remove_data_source_dumps_skips_unmount_snapshot_directories_that_are_no flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(False) flexmock(module.shutil).should_receive('rmtree').never() flexmock(module).should_receive('unmount_snapshot').never() @@ -486,12 +652,14 @@ def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_that_are_no flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/zfs_snapshots' + '/run/borgmatic/zfs_snapshots/b33f' ).and_return(True) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/zfs_snapshots/mnt/dataset' + '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' ).and_return(False) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').never() @@ -515,12 +683,14 @@ def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_after_rmtre flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/zfs_snapshots' + '/run/borgmatic/zfs_snapshots/b33f' ).and_return(True) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/zfs_snapshots/mnt/dataset' + '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' ).and_return(True).and_return(False) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').never() @@ -544,7 +714,9 @@ def test_remove_data_source_dumps_with_dry_run_skips_unmount_and_destroy(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) flexmock(module.shutil).should_receive('rmtree').never() flexmock(module).should_receive('unmount_snapshot').never() From e69c686abf65a4f46fd85b2a1a0cd43a26941243 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 21 Feb 2025 11:32:57 -0800 Subject: [PATCH 035/226] Get all unit/integration tests passing (#1001). --- borgmatic/hooks/data_source/lvm.py | 8 +- borgmatic/hooks/data_source/snapshot.py | 2 - tests/unit/actions/test_create.py | 31 +- tests/unit/borg/test_create.py | 4 +- tests/unit/hooks/data_source/test_btrfs.py | 14 +- tests/unit/hooks/data_source/test_lvm.py | 408 ++++++++++++++------- tests/unit/hooks/data_source/test_zfs.py | 116 +++--- 7 files changed, 371 insertions(+), 212 deletions(-) diff --git a/borgmatic/hooks/data_source/lvm.py b/borgmatic/hooks/data_source/lvm.py index 1d2e4b42..186234f3 100644 --- a/borgmatic/hooks/data_source/lvm.py +++ b/borgmatic/hooks/data_source/lvm.py @@ -138,7 +138,7 @@ MOUNT_POINT_HASH_LENGTH = 10 def make_borg_snapshot_pattern(pattern, logical_volume, normalized_runtime_directory): ''' Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance and a Logical_volume - contianing it, return a new Pattern with its path rewritten to be in a snapshot directory based + containing it, return a new Pattern with its path rewritten to be in a snapshot directory based on both the given runtime directory and the given Logical_volume's mount point. Move any initial caret in a regular expression pattern path to the beginning, so as not to break @@ -158,7 +158,9 @@ def make_borg_snapshot_pattern(pattern, logical_volume, normalized_runtime_direc # volumes. For instance, without this, snapshotting a logical volume at /var and another at # /var/spool would result in overlapping snapshot patterns and therefore colliding mount # attempts. - hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest(MOUNT_POINT_HASH_LENGTH), + hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest( + MOUNT_POINT_HASH_LENGTH + ), '.', # Borg 1.4+ "slashdot" hack. # Included so that the source directory ends up in the Borg archive at its "original" path. pattern.path.lstrip('^').lstrip(os.path.sep), @@ -237,7 +239,7 @@ def dump_data_sources( snapshot_mount_path = os.path.join( normalized_runtime_directory, 'lvm_snapshots', - hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest( + hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest( MOUNT_POINT_HASH_LENGTH ), logical_volume.mount_point.lstrip(os.path.sep), diff --git a/borgmatic/hooks/data_source/snapshot.py b/borgmatic/hooks/data_source/snapshot.py index b13f1aa3..ece63ce6 100644 --- a/borgmatic/hooks/data_source/snapshot.py +++ b/borgmatic/hooks/data_source/snapshot.py @@ -1,5 +1,3 @@ -import logging - import pathlib IS_A_HOOK = False diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 8ba1eed0..5e4a4c9f 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -5,16 +5,21 @@ import pytest from flexmock import flexmock from borgmatic.actions import create as module -from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type +from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type @pytest.mark.parametrize( 'pattern_line,expected_pattern', ( - ('R /foo', Pattern('/foo')), - ('P sh', Pattern('sh', Pattern_type.PATTERN_STYLE)), - ('+ /foo*', Pattern('/foo*', Pattern_type.INCLUDE)), - ('+ sh:/foo*', Pattern('/foo*', Pattern_type.INCLUDE, Pattern_style.SHELL)), + ('R /foo', Pattern('/foo', source=Pattern_source.CONFIG)), + ('P sh', Pattern('sh', Pattern_type.PATTERN_STYLE, source=Pattern_source.CONFIG)), + ('+ /foo*', Pattern('/foo*', Pattern_type.INCLUDE, source=Pattern_source.CONFIG)), + ( + '+ sh:/foo*', + Pattern( + '/foo*', Pattern_type.INCLUDE, Pattern_style.SHELL, source=Pattern_source.CONFIG + ), + ), ), ) def test_parse_pattern_transforms_pattern_line_to_instance(pattern_line, expected_pattern): @@ -28,8 +33,8 @@ def test_parse_pattern_with_invalid_pattern_line_errors(): def test_collect_patterns_converts_source_directories(): assert module.collect_patterns({'source_directories': ['/foo', '/bar']}) == ( - Pattern('/foo'), - Pattern('/bar'), + Pattern('/foo', source=Pattern_source.CONFIG), + Pattern('/bar', source=Pattern_source.CONFIG), ) @@ -48,9 +53,15 @@ def test_collect_patterns_parses_config_patterns(): def test_collect_patterns_converts_exclude_patterns(): assert module.collect_patterns({'exclude_patterns': ['/foo', '/bar', 'sh:**/baz']}) == ( - Pattern('/foo', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH), - Pattern('/bar', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH), - Pattern('**/baz', Pattern_type.NO_RECURSE, Pattern_style.SHELL), + Pattern( + '/foo', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, source=Pattern_source.CONFIG + ), + Pattern( + '/bar', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, source=Pattern_source.CONFIG + ), + Pattern( + '**/baz', Pattern_type.NO_RECURSE, Pattern_style.SHELL, source=Pattern_source.CONFIG + ), ) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 6015004a..be2ceede 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -4,7 +4,7 @@ import pytest from flexmock import flexmock from borgmatic.borg import create as module -from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type +from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type from ..test_verbosity import insert_logging_mock @@ -663,6 +663,7 @@ def test_make_base_create_command_with_stream_processes_ignores_read_special_fal '/dev/null', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, + source=Pattern_source.INTERNAL, ), ), '/run/borgmatic', @@ -713,6 +714,7 @@ def test_make_base_create_command_without_patterns_and_with_stream_processes_ign '/dev/null', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, + source=Pattern_source.INTERNAL, ), ), '/run/borgmatic', diff --git a/tests/unit/hooks/data_source/test_btrfs.py b/tests/unit/hooks/data_source/test_btrfs.py index 65c4ccbe..648902c3 100644 --- a/tests/unit/hooks/data_source/test_btrfs.py +++ b/tests/unit/hooks/data_source/test_btrfs.py @@ -1,7 +1,7 @@ import pytest from flexmock import flexmock -from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type +from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type from borgmatic.hooks.data_source import btrfs as module @@ -54,8 +54,8 @@ def test_get_subvolumes_collects_subvolumes_matching_patterns(): contained_pattern = Pattern( '/mnt1', - type=module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + type=Pattern_type.ROOT, + source=Pattern_source.CONFIG, ) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' @@ -83,8 +83,8 @@ def test_get_subvolumes_skips_non_root_patterns(): ( Pattern( '/mnt1', - type=module.borgmatic.borg.pattern.Pattern_type.EXCLUDE, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + type=Pattern_type.EXCLUDE, + source=Pattern_source.CONFIG, ), ) ) @@ -114,8 +114,8 @@ def test_get_subvolumes_skips_non_config_patterns(): ( Pattern( '/mnt1', - type=module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.HOOK, + type=Pattern_type.ROOT, + source=Pattern_source.HOOK, ), ) ) diff --git a/tests/unit/hooks/data_source/test_lvm.py b/tests/unit/hooks/data_source/test_lvm.py index 49159905..9e172de2 100644 --- a/tests/unit/hooks/data_source/test_lvm.py +++ b/tests/unit/hooks/data_source/test_lvm.py @@ -1,7 +1,7 @@ import pytest from flexmock import flexmock -from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type +from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type from borgmatic.hooks.data_source import lvm as module @@ -37,14 +37,20 @@ def test_get_logical_volumes_filters_by_patterns(): } ''' ) - contained = {Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir')} + contained = { + Pattern('/mnt/lvolume', source=Pattern_source.CONFIG), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.CONFIG), + } flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' ).with_args(None, contained).never() flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' ).with_args('/mnt/lvolume', contained).and_return( - (Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir')) + ( + Pattern('/mnt/lvolume', source=Pattern_source.CONFIG), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.CONFIG), + ) ) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' @@ -54,17 +60,116 @@ def test_get_logical_volumes_filters_by_patterns(): ).with_args('/mnt/notlvm', contained).never() assert module.get_logical_volumes( - 'lsblk', patterns=(Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir')) + 'lsblk', + patterns=( + Pattern('/mnt/lvolume', source=Pattern_source.CONFIG), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.CONFIG), + ), ) == ( module.Logical_volume( name='vgroup-lvolume', device_path='/dev/mapper/vgroup-lvolume', mount_point='/mnt/lvolume', - contained_patterns=(Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir')), + contained_patterns=( + Pattern('/mnt/lvolume', source=Pattern_source.CONFIG), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.CONFIG), + ), ), ) +def test_get_logical_volumes_skips_non_root_patterns(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( + ''' + { + "blockdevices": [ + { + "name": "vgroup-lvolume", + "path": "/dev/mapper/vgroup-lvolume", + "mountpoint": "/mnt/lvolume", + "type": "lvm" + } + ] + } + ''' + ) + contained = { + Pattern('/mnt/lvolume', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG), + Pattern('/mnt/lvolume/subdir', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG), + } + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args(None, contained).never() + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/mnt/lvolume', contained).and_return( + ( + Pattern('/mnt/lvolume', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG), + Pattern('/mnt/lvolume/subdir', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG), + ) + ) + + assert ( + module.get_logical_volumes( + 'lsblk', + patterns=( + Pattern('/mnt/lvolume', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG), + Pattern( + '/mnt/lvolume/subdir', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG + ), + ), + ) + == () + ) + + +def test_get_logical_volumes_skips_non_config_patterns(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( + ''' + { + "blockdevices": [ + { + "name": "vgroup-lvolume", + "path": "/dev/mapper/vgroup-lvolume", + "mountpoint": "/mnt/lvolume", + "type": "lvm" + } + ] + } + ''' + ) + contained = { + Pattern('/mnt/lvolume', source=Pattern_source.HOOK), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.HOOK), + } + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args(None, contained).never() + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/mnt/lvolume', contained).and_return( + ( + Pattern('/mnt/lvolume', source=Pattern_source.HOOK), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.HOOK), + ) + ) + + assert ( + module.get_logical_volumes( + 'lsblk', + patterns=( + Pattern('/mnt/lvolume', source=Pattern_source.HOOK), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.HOOK), + ), + ) + == () + ) + + def test_get_logical_volumes_with_invalid_lsblk_json_errors(): flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' @@ -138,13 +243,13 @@ def test_snapshot_logical_volume_with_non_percentage_snapshot_name_uses_lvcreate ( ( Pattern('/foo/bar/baz'), - Pattern('/run/borgmatic/lvm_snapshots/./foo/bar/baz'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./foo/bar/baz'), ), - (Pattern('/foo/bar'), Pattern('/run/borgmatic/lvm_snapshots/./foo/bar')), + (Pattern('/foo/bar'), Pattern('/run/borgmatic/lvm_snapshots/b33f/./foo/bar')), ( Pattern('^/foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION), Pattern( - '^/run/borgmatic/lvm_snapshots/./foo/bar', + '^/run/borgmatic/lvm_snapshots/b33f/./foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION, ), @@ -152,40 +257,48 @@ def test_snapshot_logical_volume_with_non_percentage_snapshot_name_uses_lvcreate ( Pattern('/foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION), Pattern( - '/run/borgmatic/lvm_snapshots/./foo/bar', + '/run/borgmatic/lvm_snapshots/b33f/./foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION, ), ), - (Pattern('/foo'), Pattern('/run/borgmatic/lvm_snapshots/./foo')), - (Pattern('/'), Pattern('/run/borgmatic/lvm_snapshots/./')), + (Pattern('/foo'), Pattern('/run/borgmatic/lvm_snapshots/b33f/./foo')), + (Pattern('/'), Pattern('/run/borgmatic/lvm_snapshots/b33f/./')), ), ) def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path( pattern, expected_pattern ): - assert module.make_borg_snapshot_pattern(pattern, '/run/borgmatic') == expected_pattern + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) + + assert ( + module.make_borg_snapshot_pattern( + pattern, flexmock(mount_point='/something'), '/run/borgmatic' + ) + == expected_pattern + ) def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns(): config = {'lvm': {}} patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')] - flexmock(module).should_receive('get_logical_volumes').and_return( - ( - module.Logical_volume( - name='lvolume1', - device_path='/dev/lvolume1', - mount_point='/mnt/lvolume1', - contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), - ), - module.Logical_volume( - name='lvolume2', - device_path='/dev/lvolume2', - mount_point='/mnt/lvolume2', - contained_patterns=(Pattern('/mnt/lvolume2'),), - ), - ) + logical_volumes = ( + module.Logical_volume( + name='lvolume1', + device_path='/dev/lvolume1', + mount_point='/mnt/lvolume1', + contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), + ), + module.Logical_volume( + name='lvolume2', + device_path='/dev/lvolume2', + mount_point='/mnt/lvolume2', + contained_patterns=(Pattern('/mnt/lvolume2'),), + ), ) + flexmock(module).should_receive('get_logical_volumes').and_return(logical_volumes) flexmock(module.os).should_receive('getpid').and_return(1234) flexmock(module).should_receive('snapshot_logical_volume').with_args( 'lvcreate', 'lvolume1_borgmatic-1234', '/dev/lvolume1', module.DEFAULT_SNAPSHOT_SIZE @@ -203,18 +316,21 @@ def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns(): ).and_return( (module.Snapshot(name='lvolume2_borgmatic-1234', device_path='/dev/lvolume2_snap'),) ) + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) flexmock(module).should_receive('mount_snapshot').with_args( - 'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume1' + 'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' ).once() flexmock(module).should_receive('mount_snapshot').with_args( - 'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume2' + 'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' ).once() flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume1/subdir'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir')) + Pattern('/mnt/lvolume1/subdir'), logical_volumes[0], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir')) flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume2'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2')) + Pattern('/mnt/lvolume2'), logical_volumes[1], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2')) assert ( module.dump_data_sources( @@ -229,8 +345,8 @@ def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns(): ) assert patterns == [ - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'), - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2'), ] @@ -259,22 +375,21 @@ def test_dump_data_sources_with_no_logical_volumes_skips_snapshots(): def test_dump_data_sources_uses_snapshot_size_for_snapshot(): config = {'lvm': {'snapshot_size': '1000PB'}} patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')] - flexmock(module).should_receive('get_logical_volumes').and_return( - ( - module.Logical_volume( - name='lvolume1', - device_path='/dev/lvolume1', - mount_point='/mnt/lvolume1', - contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), - ), - module.Logical_volume( - name='lvolume2', - device_path='/dev/lvolume2', - mount_point='/mnt/lvolume2', - contained_patterns=(Pattern('/mnt/lvolume2'),), - ), - ) + logical_volumes = ( + module.Logical_volume( + name='lvolume1', + device_path='/dev/lvolume1', + mount_point='/mnt/lvolume1', + contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), + ), + module.Logical_volume( + name='lvolume2', + device_path='/dev/lvolume2', + mount_point='/mnt/lvolume2', + contained_patterns=(Pattern('/mnt/lvolume2'),), + ), ) + flexmock(module).should_receive('get_logical_volumes').and_return(logical_volumes) flexmock(module.os).should_receive('getpid').and_return(1234) flexmock(module).should_receive('snapshot_logical_volume').with_args( 'lvcreate', @@ -298,18 +413,21 @@ def test_dump_data_sources_uses_snapshot_size_for_snapshot(): ).and_return( (module.Snapshot(name='lvolume2_borgmatic-1234', device_path='/dev/lvolume2_snap'),) ) + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) flexmock(module).should_receive('mount_snapshot').with_args( - 'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume1' + 'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' ).once() flexmock(module).should_receive('mount_snapshot').with_args( - 'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume2' + 'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' ).once() flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume1/subdir'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir')) + Pattern('/mnt/lvolume1/subdir'), logical_volumes[0], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir')) flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume2'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2')) + Pattern('/mnt/lvolume2'), logical_volumes[1], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2')) assert ( module.dump_data_sources( @@ -324,8 +442,8 @@ def test_dump_data_sources_uses_snapshot_size_for_snapshot(): ) assert patterns == [ - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'), - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2'), ] @@ -339,22 +457,21 @@ def test_dump_data_sources_uses_custom_commands(): }, } patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')] - flexmock(module).should_receive('get_logical_volumes').and_return( - ( - module.Logical_volume( - name='lvolume1', - device_path='/dev/lvolume1', - mount_point='/mnt/lvolume1', - contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), - ), - module.Logical_volume( - name='lvolume2', - device_path='/dev/lvolume2', - mount_point='/mnt/lvolume2', - contained_patterns=(Pattern('/mnt/lvolume2'),), - ), - ) + logical_volumes = ( + module.Logical_volume( + name='lvolume1', + device_path='/dev/lvolume1', + mount_point='/mnt/lvolume1', + contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), + ), + module.Logical_volume( + name='lvolume2', + device_path='/dev/lvolume2', + mount_point='/mnt/lvolume2', + contained_patterns=(Pattern('/mnt/lvolume2'),), + ), ) + flexmock(module).should_receive('get_logical_volumes').and_return(logical_volumes) flexmock(module.os).should_receive('getpid').and_return(1234) flexmock(module).should_receive('snapshot_logical_volume').with_args( '/usr/local/bin/lvcreate', @@ -378,18 +495,25 @@ def test_dump_data_sources_uses_custom_commands(): ).and_return( (module.Snapshot(name='lvolume2_borgmatic-1234', device_path='/dev/lvolume2_snap'),) ) + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) flexmock(module).should_receive('mount_snapshot').with_args( - '/usr/local/bin/mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume1' + '/usr/local/bin/mount', + '/dev/lvolume1_snap', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).once() flexmock(module).should_receive('mount_snapshot').with_args( - '/usr/local/bin/mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume2' + '/usr/local/bin/mount', + '/dev/lvolume2_snap', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).once() flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume1/subdir'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir')) + Pattern('/mnt/lvolume1/subdir'), logical_volumes[0], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir')) flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume2'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2')) + Pattern('/mnt/lvolume2'), logical_volumes[1], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2')) assert ( module.dump_data_sources( @@ -404,8 +528,8 @@ def test_dump_data_sources_uses_custom_commands(): ) assert patterns == [ - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'), - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2'), ] @@ -463,22 +587,21 @@ def test_dump_data_sources_with_dry_run_skips_snapshots_and_does_not_touch_patte def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained_patterns(): config = {'lvm': {}} patterns = [Pattern('/hmm')] - flexmock(module).should_receive('get_logical_volumes').and_return( - ( - module.Logical_volume( - name='lvolume1', - device_path='/dev/lvolume1', - mount_point='/mnt/lvolume1', - contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), - ), - module.Logical_volume( - name='lvolume2', - device_path='/dev/lvolume2', - mount_point='/mnt/lvolume2', - contained_patterns=(Pattern('/mnt/lvolume2'),), - ), - ) + logical_volumes = ( + module.Logical_volume( + name='lvolume1', + device_path='/dev/lvolume1', + mount_point='/mnt/lvolume1', + contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), + ), + module.Logical_volume( + name='lvolume2', + device_path='/dev/lvolume2', + mount_point='/mnt/lvolume2', + contained_patterns=(Pattern('/mnt/lvolume2'),), + ), ) + flexmock(module).should_receive('get_logical_volumes').and_return(logical_volumes) flexmock(module.os).should_receive('getpid').and_return(1234) flexmock(module).should_receive('snapshot_logical_volume').with_args( 'lvcreate', 'lvolume1_borgmatic-1234', '/dev/lvolume1', module.DEFAULT_SNAPSHOT_SIZE @@ -496,18 +619,21 @@ def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained ).and_return( (module.Snapshot(name='lvolume2_borgmatic-1234', device_path='/dev/lvolume2_snap'),) ) + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) flexmock(module).should_receive('mount_snapshot').with_args( - 'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume1' + 'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' ).once() flexmock(module).should_receive('mount_snapshot').with_args( - 'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume2' + 'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' ).once() flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume1/subdir'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir')) + Pattern('/mnt/lvolume1/subdir'), logical_volumes[0], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir')) flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume2'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2')) + Pattern('/mnt/lvolume2'), logical_volumes[1], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2')) assert ( module.dump_data_sources( @@ -523,8 +649,8 @@ def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained assert patterns == [ Pattern('/hmm'), - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'), - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2'), ] @@ -694,16 +820,18 @@ def test_remove_data_source_dumps_unmounts_and_remove_snapshots(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).once() flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).once() flexmock(module).should_receive('get_snapshots').and_return( ( @@ -799,9 +927,11 @@ def test_remove_data_source_dumps_with_missing_snapshot_directory_skips_unmount( flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots' + '/run/borgmatic/lvm_snapshots/b33f' ).and_return(False) flexmock(module.shutil).should_receive('rmtree').never() flexmock(module).should_receive('unmount_snapshot').never() @@ -843,24 +973,26 @@ def test_remove_data_source_dumps_with_missing_snapshot_mount_path_skips_unmount flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots' + '/run/borgmatic/lvm_snapshots/b33f' ).and_return(True) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots/mnt/lvolume1' + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' ).and_return(False) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots/mnt/lvolume2' + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' ).and_return(True) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).never() flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).once() flexmock(module).should_receive('get_snapshots').and_return( ( @@ -900,24 +1032,26 @@ def test_remove_data_source_dumps_with_successful_mount_point_removal_skips_unmo flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots' + '/run/borgmatic/lvm_snapshots/b33f' ).and_return(True) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots/mnt/lvolume1' + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' ).and_return(True).and_return(False) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots/mnt/lvolume2' + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' ).and_return(True).and_return(True) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).never() flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).once() flexmock(module).should_receive('get_snapshots').and_return( ( @@ -957,16 +1091,18 @@ def test_remove_data_source_dumps_bails_for_missing_umount_command(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).and_raise(FileNotFoundError) flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).never() flexmock(module).should_receive('get_snapshots').never() flexmock(module).should_receive('remove_snapshot').never() @@ -1000,16 +1136,18 @@ def test_remove_data_source_dumps_bails_for_umount_command_error(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).and_raise(module.subprocess.CalledProcessError(1, 'wtf')) flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).never() flexmock(module).should_receive('get_snapshots').never() flexmock(module).should_receive('remove_snapshot').never() @@ -1043,16 +1181,18 @@ def test_remove_data_source_dumps_bails_for_missing_lvs_command(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).once() flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).once() flexmock(module).should_receive('get_snapshots').and_raise(FileNotFoundError) flexmock(module).should_receive('remove_snapshot').never() @@ -1086,16 +1226,18 @@ def test_remove_data_source_dumps_bails_for_lvs_command_error(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).once() flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).once() flexmock(module).should_receive('get_snapshots').and_raise( module.subprocess.CalledProcessError(1, 'wtf') diff --git a/tests/unit/hooks/data_source/test_zfs.py b/tests/unit/hooks/data_source/test_zfs.py index 7bc07122..5900e037 100644 --- a/tests/unit/hooks/data_source/test_zfs.py +++ b/tests/unit/hooks/data_source/test_zfs.py @@ -3,7 +3,7 @@ import os import pytest from flexmock import flexmock -from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type +from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type from borgmatic.hooks.data_source import zfs as module @@ -19,8 +19,8 @@ def test_get_datasets_to_backup_filters_datasets_by_patterns(): ( Pattern( '/dataset', - module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + Pattern_type.ROOT, + source=Pattern_source.CONFIG, ), ) ) @@ -33,18 +33,18 @@ def test_get_datasets_to_backup_filters_datasets_by_patterns(): patterns=( Pattern( '/foo', - module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + Pattern_type.ROOT, + source=Pattern_source.CONFIG, ), Pattern( '/dataset', - module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + Pattern_type.ROOT, + source=Pattern_source.CONFIG, ), Pattern( '/bar', - module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + Pattern_type.ROOT, + source=Pattern_source.CONFIG, ), ), ) == ( @@ -54,8 +54,8 @@ def test_get_datasets_to_backup_filters_datasets_by_patterns(): contained_patterns=( Pattern( '/dataset', - module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + Pattern_type.ROOT, + source=Pattern_source.CONFIG, ), ), ), @@ -74,8 +74,8 @@ def test_get_datasets_to_backup_skips_non_root_patterns(): ( Pattern( '/dataset', - module.borgmatic.borg.pattern.Pattern_type.EXCLUDE, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + Pattern_type.EXCLUDE, + source=Pattern_source.CONFIG, ), ) ) @@ -83,26 +83,29 @@ def test_get_datasets_to_backup_skips_non_root_patterns(): 'get_contained_patterns' ).with_args('/other', object).and_return(()) - assert module.get_datasets_to_backup( - 'zfs', - patterns=( - Pattern( - '/foo', - module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + assert ( + module.get_datasets_to_backup( + 'zfs', + patterns=( + Pattern( + '/foo', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), + Pattern( + '/dataset', + Pattern_type.EXCLUDE, + source=Pattern_source.CONFIG, + ), + Pattern( + '/bar', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), ), - Pattern( - '/dataset', - module.borgmatic.borg.pattern.Pattern_type.EXCLUDE, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, - ), - Pattern( - '/bar', - module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, - ), - ), - ) == () + ) + == () + ) def test_get_datasets_to_backup_skips_non_config_patterns(): @@ -117,8 +120,8 @@ def test_get_datasets_to_backup_skips_non_config_patterns(): ( Pattern( '/dataset', - module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.HOOK, + Pattern_type.ROOT, + source=Pattern_source.HOOK, ), ) ) @@ -126,26 +129,29 @@ def test_get_datasets_to_backup_skips_non_config_patterns(): 'get_contained_patterns' ).with_args('/other', object).and_return(()) - assert module.get_datasets_to_backup( - 'zfs', - patterns=( - Pattern( - '/foo', - module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, + assert ( + module.get_datasets_to_backup( + 'zfs', + patterns=( + Pattern( + '/foo', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), + Pattern( + '/dataset', + Pattern_type.ROOT, + source=Pattern_source.HOOK, + ), + Pattern( + '/bar', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), ), - Pattern( - '/dataset', - module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.HOOK, - ), - Pattern( - '/bar', - module.borgmatic.borg.pattern.Pattern_type.ROOT, - source=module.borgmatic.borg.pattern.Pattern_source.CONFIG, - ), - ), - ) == () + ) + == () + ) def test_get_datasets_to_backup_filters_datasets_by_user_property(): @@ -169,9 +175,7 @@ def test_get_datasets_to_backup_filters_datasets_by_user_property(): name='dataset', mount_point='/dataset', auto_backup=True, - contained_patterns=( - Pattern('/dataset', source=module.borgmatic.borg.pattern.Pattern_source.HOOK), - ), + contained_patterns=(Pattern('/dataset', source=Pattern_source.HOOK),), ), ) From bc25ac4eea1cde314d5cc34e9fafdc2070fb1360 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 21 Feb 2025 16:32:07 -0800 Subject: [PATCH 036/226] Fix Btrfs end-to-end-test (#1001). --- borgmatic/hooks/data_source/btrfs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borgmatic/hooks/data_source/btrfs.py b/borgmatic/hooks/data_source/btrfs.py index b81a9b05..ec6d33df 100644 --- a/borgmatic/hooks/data_source/btrfs.py +++ b/borgmatic/hooks/data_source/btrfs.py @@ -125,10 +125,10 @@ def make_snapshot_exclude_pattern(subvolume_path): # pragma: no cover snapshot_directory, subvolume_path.lstrip(os.path.sep), snapshot_directory, - source=borgmatic.borg.pattern.Pattern_source.HOOK, ), borgmatic.borg.pattern.Pattern_type.NO_RECURSE, borgmatic.borg.pattern.Pattern_style.FNMATCH, + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) From a3e34d63e9a23863ad86317c544c75a299855544 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 21 Feb 2025 16:36:12 -0800 Subject: [PATCH 037/226] Remove debugging prints (#1001). --- borgmatic/hooks/data_source/zfs.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/borgmatic/hooks/data_source/zfs.py b/borgmatic/hooks/data_source/zfs.py index 61dfeb22..66cc4dce 100644 --- a/borgmatic/hooks/data_source/zfs.py +++ b/borgmatic/hooks/data_source/zfs.py @@ -374,7 +374,6 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d umount_command = hook_config.get('umount_command', 'umount') for snapshots_directory in glob.glob(snapshots_glob): - print('*** snapshots_glob:', snapshots_glob, 'snapshots_directory:', snapshots_directory) if not os.path.isdir(snapshots_directory): continue @@ -382,7 +381,6 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d # child datasets before the shorter mount point paths of parent datasets. for mount_point in reversed(dataset_mount_points): snapshot_mount_path = os.path.join(snapshots_directory, mount_point.lstrip(os.path.sep)) - print('*** snapshot_mount_path:', snapshot_mount_path) if not os.path.isdir(snapshot_mount_path): continue From 9ba78fa33bdefb44b4aa01977796413b3aed1c41 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 21 Feb 2025 17:59:45 -0800 Subject: [PATCH 038/226] Don't try to unmount empty directories (#1001). --- borgmatic/hooks/data_source/lvm.py | 7 +- borgmatic/hooks/data_source/zfs.py | 7 +- tests/unit/hooks/data_source/test_lvm.py | 87 ++++++++++++++++++++++-- tests/unit/hooks/data_source/test_zfs.py | 41 +++++++++++ 4 files changed, 134 insertions(+), 8 deletions(-) diff --git a/borgmatic/hooks/data_source/lvm.py b/borgmatic/hooks/data_source/lvm.py index 186234f3..11cd1c09 100644 --- a/borgmatic/hooks/data_source/lvm.py +++ b/borgmatic/hooks/data_source/lvm.py @@ -376,7 +376,10 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d snapshot_mount_path = os.path.join( snapshots_directory, logical_volume.mount_point.lstrip(os.path.sep) ) - if not os.path.isdir(snapshot_mount_path): + + # If the snapshot mount path is empty, this is probably just a "shadow" of a nested + # logical volume and therefore there's nothing to unmount. + if not os.path.isdir(snapshot_mount_path) or not os.listdir(snapshot_mount_path): continue # This might fail if the directory is already mounted, but we swallow errors here since @@ -401,7 +404,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d return except subprocess.CalledProcessError as error: logger.debug(error) - return + continue if not dry_run: shutil.rmtree(snapshots_directory) diff --git a/borgmatic/hooks/data_source/zfs.py b/borgmatic/hooks/data_source/zfs.py index 66cc4dce..a7e9f01d 100644 --- a/borgmatic/hooks/data_source/zfs.py +++ b/borgmatic/hooks/data_source/zfs.py @@ -381,7 +381,10 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d # child datasets before the shorter mount point paths of parent datasets. for mount_point in reversed(dataset_mount_points): snapshot_mount_path = os.path.join(snapshots_directory, mount_point.lstrip(os.path.sep)) - if not os.path.isdir(snapshot_mount_path): + + # If the snapshot mount path is empty, this is probably just a "shadow" of a nested + # dataset and therefore there's nothing to unmount. + if not os.path.isdir(snapshot_mount_path) or not os.listdir(snapshot_mount_path): continue # This might fail if the path is already mounted, but we swallow errors here since we'll @@ -408,7 +411,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d continue if not dry_run: - shutil.rmtree(snapshots_directory, ignore_errors=True) + shutil.rmtree(snapshots_directory) # Destroy snapshots. full_snapshot_names = get_all_snapshots(zfs_command) diff --git a/tests/unit/hooks/data_source/test_lvm.py b/tests/unit/hooks/data_source/test_lvm.py index 9e172de2..34cf5250 100644 --- a/tests/unit/hooks/data_source/test_lvm.py +++ b/tests/unit/hooks/data_source/test_lvm.py @@ -824,6 +824,7 @@ def test_remove_data_source_dumps_unmounts_and_remove_snapshots(): lambda path: [path.replace('*', 'b33f')] ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', @@ -985,6 +986,72 @@ def test_remove_data_source_dumps_with_missing_snapshot_mount_path_skips_unmount flexmock(module.os.path).should_receive('isdir').with_args( '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' ).and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) + flexmock(module.shutil).should_receive('rmtree') + flexmock(module).should_receive('unmount_snapshot').with_args( + 'umount', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', + ).never() + flexmock(module).should_receive('unmount_snapshot').with_args( + 'umount', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', + ).once() + flexmock(module).should_receive('get_snapshots').and_return( + ( + module.Snapshot('lvolume1_borgmatic-1234', '/dev/lvolume1'), + module.Snapshot('lvolume2_borgmatic-1234', '/dev/lvolume2'), + ), + ) + flexmock(module).should_receive('remove_snapshot').with_args('lvremove', '/dev/lvolume1').once() + flexmock(module).should_receive('remove_snapshot').with_args('lvremove', '/dev/lvolume2').once() + + module.remove_data_source_dumps( + hook_config=config['lvm'], + config=config, + borgmatic_runtime_directory='/run/borgmatic', + dry_run=False, + ) + + +def test_remove_data_source_dumps_with_empty_snapshot_mount_path_skips_unmount(): + config = {'lvm': {}} + flexmock(module).should_receive('get_logical_volumes').and_return( + ( + module.Logical_volume( + name='lvolume1', + device_path='/dev/lvolume1', + mount_point='/mnt/lvolume1', + contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), + ), + module.Logical_volume( + name='lvolume2', + device_path='/dev/lvolume2', + mount_point='/mnt/lvolume2', + contained_patterns=(Pattern('/mnt/lvolume2'),), + ), + ) + ) + flexmock(module.borgmatic.config.paths).should_receive( + 'replace_temporary_subdirectory_with_glob' + ).and_return('/run/borgmatic') + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) + flexmock(module.os.path).should_receive('isdir').with_args( + '/run/borgmatic/lvm_snapshots/b33f' + ).and_return(True) + flexmock(module.os.path).should_receive('isdir').with_args( + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' + ).and_return(True) + flexmock(module.os).should_receive('listdir').with_args( + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' + ).and_return([]) + flexmock(module.os.path).should_receive('isdir').with_args( + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' + ).and_return(True) + flexmock(module.os).should_receive('listdir').with_args( + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' + ).and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', @@ -1044,6 +1111,7 @@ def test_remove_data_source_dumps_with_successful_mount_point_removal_skips_unmo flexmock(module.os.path).should_receive('isdir').with_args( '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' ).and_return(True).and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', @@ -1095,6 +1163,7 @@ def test_remove_data_source_dumps_bails_for_missing_umount_command(): lambda path: [path.replace('*', 'b33f')] ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', @@ -1115,7 +1184,7 @@ def test_remove_data_source_dumps_bails_for_missing_umount_command(): ) -def test_remove_data_source_dumps_bails_for_umount_command_error(): +def test_remove_data_source_dumps_swallows_umount_command_error(): config = {'lvm': {}} flexmock(module).should_receive('get_logical_volumes').and_return( ( @@ -1140,6 +1209,7 @@ def test_remove_data_source_dumps_bails_for_umount_command_error(): lambda path: [path.replace('*', 'b33f')] ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', @@ -1148,9 +1218,15 @@ def test_remove_data_source_dumps_bails_for_umount_command_error(): flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', - ).never() - flexmock(module).should_receive('get_snapshots').never() - flexmock(module).should_receive('remove_snapshot').never() + ).once() + flexmock(module).should_receive('get_snapshots').and_return( + ( + module.Snapshot('lvolume1_borgmatic-1234', '/dev/lvolume1'), + module.Snapshot('lvolume2_borgmatic-1234', '/dev/lvolume2'), + ), + ) + flexmock(module).should_receive('remove_snapshot').with_args('lvremove', '/dev/lvolume1').once() + flexmock(module).should_receive('remove_snapshot').with_args('lvremove', '/dev/lvolume2').once() module.remove_data_source_dumps( hook_config=config['lvm'], @@ -1185,6 +1261,7 @@ def test_remove_data_source_dumps_bails_for_missing_lvs_command(): lambda path: [path.replace('*', 'b33f')] ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', @@ -1230,6 +1307,7 @@ def test_remove_data_source_dumps_bails_for_lvs_command_error(): lambda path: [path.replace('*', 'b33f')] ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', @@ -1275,6 +1353,7 @@ def test_remove_data_source_with_dry_run_skips_snapshot_unmount_and_delete(): ).and_return('/run/borgmatic') flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree').never() flexmock(module).should_receive('unmount_snapshot').never() flexmock(module).should_receive('get_snapshots').and_return( diff --git a/tests/unit/hooks/data_source/test_zfs.py b/tests/unit/hooks/data_source/test_zfs.py index 5900e037..fbe18fa9 100644 --- a/tests/unit/hooks/data_source/test_zfs.py +++ b/tests/unit/hooks/data_source/test_zfs.py @@ -477,6 +477,7 @@ def test_remove_data_source_dumps_unmounts_and_destroys_snapshots(): lambda path: [path.replace('*', 'b33f')] ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' @@ -505,6 +506,7 @@ def test_remove_data_source_dumps_use_custom_commands(): lambda path: [path.replace('*', 'b33f')] ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( '/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' @@ -580,6 +582,7 @@ def test_remove_data_source_dumps_bails_for_missing_umount_command(): lambda path: [path.replace('*', 'b33f')] ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( '/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' @@ -605,6 +608,7 @@ def test_remove_data_source_dumps_swallows_umount_command_error(): lambda path: [path.replace('*', 'b33f')] ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( '/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' @@ -665,6 +669,41 @@ def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_that_are_no flexmock(module.os.path).should_receive('isdir').with_args( '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' ).and_return(False) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) + flexmock(module.shutil).should_receive('rmtree') + flexmock(module).should_receive('unmount_snapshot').never() + flexmock(module).should_receive('get_all_snapshots').and_return( + ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid') + ) + flexmock(module).should_receive('destroy_snapshot').with_args( + 'zfs', 'dataset@borgmatic-1234' + ).once() + + module.remove_data_source_dumps( + hook_config={}, + config={'source_directories': '/mnt/dataset', 'zfs': {}}, + borgmatic_runtime_directory='/run/borgmatic', + dry_run=False, + ) + + +def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_that_are_empty(): + flexmock(module).should_receive('get_all_dataset_mount_points').and_return(('/mnt/dataset',)) + flexmock(module.borgmatic.config.paths).should_receive( + 'replace_temporary_subdirectory_with_glob' + ).and_return('/run/borgmatic') + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) + flexmock(module.os.path).should_receive('isdir').with_args( + '/run/borgmatic/zfs_snapshots/b33f' + ).and_return(True) + flexmock(module.os.path).should_receive('isdir').with_args( + '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' + ).and_return(True) + flexmock(module.os).should_receive('listdir').with_args( + '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' + ).and_return([]) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').never() flexmock(module).should_receive('get_all_snapshots').and_return( @@ -696,6 +735,7 @@ def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_after_rmtre flexmock(module.os.path).should_receive('isdir').with_args( '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' ).and_return(True).and_return(False) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').never() flexmock(module).should_receive('get_all_snapshots').and_return( @@ -722,6 +762,7 @@ def test_remove_data_source_dumps_with_dry_run_skips_unmount_and_destroy(): lambda path: [path.replace('*', 'b33f')] ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree').never() flexmock(module).should_receive('unmount_snapshot').never() flexmock(module).should_receive('get_all_snapshots').and_return( From 02d8ecd66ee99bc796c0e116d59aa93d7bcbfad9 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 21 Feb 2025 18:08:34 -0800 Subject: [PATCH 039/226] Document the root pattern requirement for snapshotting (#1001). --- docs/how-to/snapshot-your-filesystems.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/how-to/snapshot-your-filesystems.md b/docs/how-to/snapshot-your-filesystems.md index 36078ead..f1b6f8e2 100644 --- a/docs/how-to/snapshot-your-filesystems.md +++ b/docs/how-to/snapshot-your-filesystems.md @@ -54,8 +54,8 @@ You have a couple of options for borgmatic to find and backup your ZFS datasets: * For any dataset you'd like backed up, add its mount point to borgmatic's `source_directories` option. * New in version 1.9.6 Or - include the mount point with borgmatic's `patterns` or `patterns_from` - options. + include the mount point as a root pattern with borgmatic's `patterns` or + `patterns_from` options. * Or set the borgmatic-specific user property `org.torsion.borgmatic:backup=auto` onto your dataset, e.g. by running `zfs set org.torsion.borgmatic:backup=auto datasetname`. Then borgmatic can find @@ -152,7 +152,8 @@ For any subvolume you'd like backed up, add its path to borgmatic's `source_directories` option. New in version 1.9.6 Or include -the mount point with borgmatic's `patterns` or `patterns_from` options. +the mount point as a root pattern with borgmatic's `patterns` or `patterns_from` +options. During a backup, borgmatic snapshots these subvolumes (non-recursively) and includes the snapshotted files in the paths sent to Borg. borgmatic is also @@ -257,7 +258,8 @@ For any logical volume you'd like backed up, add its mount point to borgmatic's `source_directories` option. New in version 1.9.6 Or include -the mount point with borgmatic's `patterns` or `patterns_from` options. +the mount point as a root pattern with borgmatic's `patterns` or `patterns_from` +options. During a backup, borgmatic automatically snapshots these discovered logical volumes (non-recursively), temporarily mounts the snapshots within its [runtime From 2280bb26b69077a5140a9b8265e6e9161c56e640 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 21 Feb 2025 22:08:08 -0800 Subject: [PATCH 040/226] Fix a few tests to mock more accurately. --- borgmatic/borg/environment.py | 6 ++-- tests/unit/borg/test_environment.py | 48 ++++++++++++++++------------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/borgmatic/borg/environment.py b/borgmatic/borg/environment.py index 273fa91b..44d764e1 100644 --- a/borgmatic/borg/environment.py +++ b/borgmatic/borg/environment.py @@ -27,7 +27,7 @@ DEFAULT_BOOL_OPTION_TO_UPPERCASE_ENVIRONMENT_VARIABLE = { def make_environment(config): ''' Given a borgmatic configuration dict, convert it to a Borg environment variable dict, merge it - with a copy of the current environment variables and return the result. + with a copy of the current environment variables, and return the result. Do not reuse this environment across multiple Borg invocations, because it can include references to resources like anonymous pipes for passphrases—which can only be consumed once. @@ -40,8 +40,8 @@ def make_environment(config): In borgmatic, we want to simulate this precedence order, but there are some additional complications. First, values can come from either configuration or from environment variables - set outside borgmatic and configured options should take precedence. Second, when borgmatic gets - a passphrase—directly from configuration or indirectly via a credential hook or a passcommand—we + set outside borgmatic; configured options should take precedence. Second, when borgmatic gets a + passphrase—directly from configuration or indirectly via a credential hook or a passcommand—we want to pass that passphrase to Borg via an anonymous pipe (+ BORG_PASSPHRASE_FD), since that's more secure than using an environment variable (BORG_PASSPHRASE). ''' diff --git a/tests/unit/borg/test_environment.py b/tests/unit/borg/test_environment.py index 28ce3446..739aeb48 100644 --- a/tests/unit/borg/test_environment.py +++ b/tests/unit/borg/test_environment.py @@ -5,6 +5,9 @@ from borgmatic.borg import environment as module def test_make_environment_with_passcommand_should_call_it_and_set_passphrase_file_descriptor_in_environment(): flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).and_return(None) flexmock(module.borgmatic.borg.passcommand).should_receive( 'get_passphrase_from_passcommand' ).and_return('passphrase') @@ -21,12 +24,12 @@ def test_make_environment_with_passcommand_should_call_it_and_set_passphrase_fil def test_make_environment_with_passphrase_should_set_passphrase_file_descriptor_in_environment(): flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) - flexmock(module.borgmatic.borg.passcommand).should_receive( - 'get_passphrase_from_passcommand' - ).and_return(None) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.borg.passcommand).should_receive( + 'get_passphrase_from_passcommand' + ).and_return(None) flexmock(module.os).should_receive('pipe').and_return((3, 4)) flexmock(module.os).should_receive('write') flexmock(module.os).should_receive('close') @@ -41,12 +44,12 @@ def test_make_environment_with_passphrase_should_set_passphrase_file_descriptor_ def test_make_environment_with_credential_tag_passphrase_should_load_it_and_set_passphrase_file_descriptor_in_environment(): flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) config = {'encryption_passphrase': '{credential systemd pass}'} - flexmock(module.borgmatic.borg.passcommand).should_receive( - 'get_passphrase_from_passcommand' - ).and_return(None) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential', ).with_args('{credential systemd pass}', config).and_return('pass') + flexmock(module.borgmatic.borg.passcommand).should_receive( + 'get_passphrase_from_passcommand' + ).never() flexmock(module.os).should_receive('pipe').and_return((3, 4)) flexmock(module.os).should_receive('write') flexmock(module.os).should_receive('close') @@ -60,8 +63,8 @@ def test_make_environment_with_credential_tag_passphrase_should_load_it_and_set_ def test_make_environment_with_ssh_command_should_set_environment(): flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) - flexmock(module.borgmatic.borg.passcommand).should_receive( - 'get_passphrase_from_passcommand' + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' ).and_return(None) flexmock(module.os).should_receive('pipe').never() environment = module.make_environment({'ssh_command': 'ssh -C'}) @@ -71,8 +74,8 @@ def test_make_environment_with_ssh_command_should_set_environment(): def test_make_environment_without_configuration_sets_certain_environment_variables(): flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) - flexmock(module.borgmatic.borg.passcommand).should_receive( - 'get_passphrase_from_passcommand' + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' ).and_return(None) flexmock(module.os).should_receive('pipe').never() environment = module.make_environment({}) @@ -94,8 +97,8 @@ def test_make_environment_without_configuration_passes_through_default_environme 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK': 'nah', } ) - flexmock(module.borgmatic.borg.passcommand).should_receive( - 'get_passphrase_from_passcommand' + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' ).and_return(None) flexmock(module.os).should_receive('pipe').never() environment = module.make_environment({}) @@ -110,8 +113,8 @@ def test_make_environment_without_configuration_passes_through_default_environme def test_make_environment_with_relocated_repo_access_true_should_set_environment_yes(): flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) - flexmock(module.borgmatic.borg.passcommand).should_receive( - 'get_passphrase_from_passcommand' + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' ).and_return(None) flexmock(module.os).should_receive('pipe').never() environment = module.make_environment({'relocated_repo_access_is_ok': True}) @@ -121,8 +124,8 @@ def test_make_environment_with_relocated_repo_access_true_should_set_environment def test_make_environment_with_relocated_repo_access_false_should_set_environment_no(): flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) - flexmock(module.borgmatic.borg.passcommand).should_receive( - 'get_passphrase_from_passcommand' + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' ).and_return(None) flexmock(module.os).should_receive('pipe').never() environment = module.make_environment({'relocated_repo_access_is_ok': False}) @@ -132,8 +135,8 @@ def test_make_environment_with_relocated_repo_access_false_should_set_environmen def test_make_environment_check_i_know_what_i_am_doing_true_should_set_environment_YES(): flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) - flexmock(module.borgmatic.borg.passcommand).should_receive( - 'get_passphrase_from_passcommand' + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' ).and_return(None) flexmock(module.os).should_receive('pipe').never() environment = module.make_environment({'check_i_know_what_i_am_doing': True}) @@ -143,8 +146,8 @@ def test_make_environment_check_i_know_what_i_am_doing_true_should_set_environme def test_make_environment_check_i_know_what_i_am_doing_false_should_set_environment_NO(): flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) - flexmock(module.borgmatic.borg.passcommand).should_receive( - 'get_passphrase_from_passcommand' + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' ).and_return(None) flexmock(module.os).should_receive('pipe').never() environment = module.make_environment({'check_i_know_what_i_am_doing': False}) @@ -154,9 +157,10 @@ def test_make_environment_check_i_know_what_i_am_doing_false_should_set_environm def test_make_environment_with_integer_variable_value(): flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) - flexmock(module.borgmatic.borg.passcommand).should_receive( - 'get_passphrase_from_passcommand' + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' ).and_return(None) flexmock(module.os).should_receive('pipe').never() environment = module.make_environment({'borg_files_cache_ttl': 40}) + assert environment.get('BORG_FILES_CACHE_TTL') == '40' From a61eba8c79caa53f8ab2d4efedd3748033c39828 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 21 Feb 2025 22:30:31 -0800 Subject: [PATCH 041/226] Add PR number to NEWS item. --- NEWS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 7e78180e..357eed09 100644 --- a/NEWS +++ b/NEWS @@ -4,14 +4,14 @@ https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/ * #996: Fix the "create" action to omit the repository label prefix from Borg's output when databases are enabled. + * #998: Send the "encryption_passphrase" option to Borg via an anonymous pipe, which is more secure + than using an environment variable. * #1001: For the ZFS, Btrfs, and LVM hooks, only make snapshots for root patterns that come from a borgmatic configuration option (e.g. "source_directories")—not from other hooks within borgmatic. * #1001: Fix a ZFS/LVM error due to colliding snapshot mount points for nested datasets or logical volumes. * #1001: Don't try to snapshot ZFS datasets that have the "canmount=off" property. - * Send the "encryption_passphrase" option to Borg via an anonymous pipe, which is more secure than - using an environment variable. * Fix another error in the Btrfs hook when a subvolume mounted at "/" is configured in borgmatic's source directories. From 34bb09e9be0e476f95b2c5cbe004a94c63277c8c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Feb 2025 09:26:08 -0800 Subject: [PATCH 042/226] Document Zabbix server version compatibility (#1003). --- docs/how-to/monitor-your-backups.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/how-to/monitor-your-backups.md b/docs/how-to/monitor-your-backups.md index 43acdb08..ff68f668 100644 --- a/docs/how-to/monitor-your-backups.md +++ b/docs/how-to/monitor-your-backups.md @@ -710,7 +710,8 @@ zabbix: - fail ``` -This hook requires the Zabbix server be running version 7.0+ +This hook requires the Zabbix server be running version 7.0. ([Support for Zabbix +7.2+](https://projects.torsion.org/borgmatic-collective/borgmatic/issues/1003) is planned.) ### Authentication methods From ad3392ca1537acbd3149b3a1c5ec66b63f705597 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Feb 2025 09:55:07 -0800 Subject: [PATCH 043/226] Ignore the BORG_PASSCOMMAND environment variable when the "encryption_passphase" option is set. --- borgmatic/borg/environment.py | 1 + tests/unit/borg/test_environment.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/borgmatic/borg/environment.py b/borgmatic/borg/environment.py index 44d764e1..19c8ad6e 100644 --- a/borgmatic/borg/environment.py +++ b/borgmatic/borg/environment.py @@ -55,6 +55,7 @@ def make_environment(config): if 'encryption_passphrase' in config: environment.pop('BORG_PASSPHRASE', None) + environment.pop('BORG_PASSCOMMAND', None) if 'encryption_passcommand' in config: environment.pop('BORG_PASSCOMMAND', None) diff --git a/tests/unit/borg/test_environment.py b/tests/unit/borg/test_environment.py index 739aeb48..46bb33b7 100644 --- a/tests/unit/borg/test_environment.py +++ b/tests/unit/borg/test_environment.py @@ -4,7 +4,9 @@ from borgmatic.borg import environment as module def test_make_environment_with_passcommand_should_call_it_and_set_passphrase_file_descriptor_in_environment(): - flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module.os).should_receive('environ').and_return( + {'USER': 'root', 'BORG_PASSCOMMAND': 'nope'} + ) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).and_return(None) @@ -18,12 +20,15 @@ def test_make_environment_with_passcommand_should_call_it_and_set_passphrase_fil environment = module.make_environment({'encryption_passcommand': 'command'}) + assert environment.get('BORG_PASSPHRASE') is None assert environment.get('BORG_PASSCOMMAND') is None assert environment.get('BORG_PASSPHRASE_FD') == '3' def test_make_environment_with_passphrase_should_set_passphrase_file_descriptor_in_environment(): - flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module.os).should_receive('environ').and_return( + {'USER': 'root', 'BORG_PASSPHRASE': 'nope', 'BORG_PASSCOMMAND': 'nope'} + ) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) @@ -38,6 +43,7 @@ def test_make_environment_with_passphrase_should_set_passphrase_file_descriptor_ environment = module.make_environment({'encryption_passphrase': 'pass'}) assert environment.get('BORG_PASSPHRASE') is None + assert environment.get('BORG_PASSCOMMAND') is None assert environment.get('BORG_PASSPHRASE_FD') == '3' From 5d9c111910ce21c18b83407f34b6e38d2c56cfe0 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Feb 2025 14:26:21 -0800 Subject: [PATCH 044/226] Fix a runtime directory error from a conflict between "extra_borg_options" and special file detection (#999). --- NEWS | 2 ++ borgmatic/borg/create.py | 5 ++-- borgmatic/borg/flags.py | 41 ++++++++++++++++++++++++++ tests/unit/borg/test_create.py | 54 +++++++++++++++++++--------------- tests/unit/borg/test_flags.py | 32 ++++++++++++++++++++ 5 files changed, 108 insertions(+), 26 deletions(-) diff --git a/NEWS b/NEWS index 357eed09..cc905bba 100644 --- a/NEWS +++ b/NEWS @@ -6,6 +6,8 @@ databases are enabled. * #998: Send the "encryption_passphrase" option to Borg via an anonymous pipe, which is more secure than using an environment variable. + * #999: Fix a runtime directory error from a conflict between "extra_borg_options" and special file + detection. * #1001: For the ZFS, Btrfs, and LVM hooks, only make snapshots for root patterns that come from a borgmatic configuration option (e.g. "source_directories")—not from other hooks within borgmatic. diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index aea0cbc8..54a7f9ad 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -140,9 +140,10 @@ def collect_special_file_paths( consume any database dumps and therefore borgmatic will hang when it tries to do so. ''' # Omit "--exclude-nodump" from the Borg dry run command, because that flag causes Borg to open - # files including any named pipe we've created. + # files including any named pipe we've created. And omit "--filter" because that can break the + # paths output parsing below such that path lines no longer start with th expected "- ". paths_output = execute_command_and_capture_output( - tuple(argument for argument in create_command if argument != '--exclude-nodump') + flags.omit_flag_and_value(flags.omit_flag(create_command, '--exclude-nodump'), '--filter') + ('--dry-run', '--list'), capture_stderr=True, working_directory=working_directory, diff --git a/borgmatic/borg/flags.py b/borgmatic/borg/flags.py index 97bda27d..a88400e1 100644 --- a/borgmatic/borg/flags.py +++ b/borgmatic/borg/flags.py @@ -156,3 +156,44 @@ def warn_for_aggressive_archive_flags(json_command, json_output): logger.debug(f'Cannot parse JSON output from archive command: {error}') except (TypeError, KeyError): logger.debug('Cannot parse JSON output from archive command: No "archives" key found') + + +def omit_flag(arguments, flag): + ''' + Given a sequence of Borg command-line arguments, return them with the given (valueless) flag + omitted. For instance, if the flag is "--flag" and arguments is: + + ('borg', 'create', '--flag', '--other-flag') + + ... then return: + + ('borg', 'create', '--other-flag') + ''' + return tuple(argument for argument in arguments if argument != flag) + + +def omit_flag_and_value(arguments, flag): + ''' + Given a sequence of Borg command-line arguments, return them with the given flag and its + corresponding value omitted. For instance, if the flag is "--flag" and arguments is: + + ('borg', 'create', '--flag', 'value', '--other-flag') + + ... or: + + ('borg', 'create', '--flag=value', '--other-flag') + + ... then return: + + ('borg', 'create', '--other-flag') + ''' + # This works by zipping together a list of overlapping pairwise arguments. E.g., ('one', 'two', + # 'three', 'four') becomes ((None, 'one'), ('one, 'two'), ('two', 'three'), ('three', 'four')). + # This makes it easy to "look back" at the previous arguments so we can exclude both a flag and + # its value. + return tuple( + argument + for (previous_argument, argument) in zip((None,) + arguments, arguments) + if flag not in (previous_argument, argument) + if not argument.startswith(f'{flag}=') + ) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 3f1f09a7..ad7c044e 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -185,6 +185,12 @@ def test_any_parent_directories_treats_unrelated_paths_as_non_match(): def test_collect_special_file_paths_parses_special_files_from_borg_dry_run_file_list(): + flexmock(module.flags).should_receive('omit_flag').replace_with( + lambda arguments, flag: arguments + ) + flexmock(module.flags).should_receive('omit_flag_and_value').replace_with( + lambda arguments, flag: arguments + ) flexmock(module.environment).should_receive('make_environment').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').and_return( 'Processing files ...\n- /foo\n+ /bar\n- /baz' @@ -204,6 +210,12 @@ def test_collect_special_file_paths_parses_special_files_from_borg_dry_run_file_ def test_collect_special_file_paths_skips_borgmatic_runtime_directory(): + flexmock(module.flags).should_receive('omit_flag').replace_with( + lambda arguments, flag: arguments + ) + flexmock(module.flags).should_receive('omit_flag_and_value').replace_with( + lambda arguments, flag: arguments + ) flexmock(module.environment).should_receive('make_environment').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').and_return( '+ /foo\n- /run/borgmatic/bar\n- /baz' @@ -231,6 +243,12 @@ def test_collect_special_file_paths_skips_borgmatic_runtime_directory(): def test_collect_special_file_paths_with_borgmatic_runtime_directory_missing_from_paths_output_errors(): + flexmock(module.flags).should_receive('omit_flag').replace_with( + lambda arguments, flag: arguments + ) + flexmock(module.flags).should_receive('omit_flag_and_value').replace_with( + lambda arguments, flag: arguments + ) flexmock(module.environment).should_receive('make_environment').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').and_return( '+ /foo\n- /bar\n- /baz' @@ -251,6 +269,12 @@ def test_collect_special_file_paths_with_borgmatic_runtime_directory_missing_fro def test_collect_special_file_paths_with_dry_run_and_borgmatic_runtime_directory_missing_from_paths_output_does_not_raise(): + flexmock(module.flags).should_receive('omit_flag').replace_with( + lambda arguments, flag: arguments + ) + flexmock(module.flags).should_receive('omit_flag_and_value').replace_with( + lambda arguments, flag: arguments + ) flexmock(module.environment).should_receive('make_environment').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').and_return( '+ /foo\n- /bar\n- /baz' @@ -270,6 +294,12 @@ def test_collect_special_file_paths_with_dry_run_and_borgmatic_runtime_directory def test_collect_special_file_paths_excludes_non_special_files(): + flexmock(module.flags).should_receive('omit_flag').replace_with( + lambda arguments, flag: arguments + ) + flexmock(module.flags).should_receive('omit_flag_and_value').replace_with( + lambda arguments, flag: arguments + ) flexmock(module.environment).should_receive('make_environment').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').and_return( '+ /foo\n+ /bar\n+ /baz' @@ -290,30 +320,6 @@ def test_collect_special_file_paths_excludes_non_special_files(): ) == ('/foo', '/baz') -def test_collect_special_file_paths_omits_exclude_no_dump_flag_from_command(): - flexmock(module.environment).should_receive('make_environment').and_return(None) - flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'create', '--dry-run', '--list'), - capture_stderr=True, - working_directory=None, - environment=None, - borg_local_path='borg', - borg_exit_codes=None, - ).and_return('Processing files ...\n- /foo\n+ /bar\n- /baz').once() - flexmock(module).should_receive('special_file').and_return(True) - flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module).should_receive('any_parent_directories').never() - - module.collect_special_file_paths( - dry_run=False, - create_command=('borg', 'create', '--exclude-nodump'), - config={}, - local_path='borg', - working_directory=None, - borgmatic_runtime_directory='/run/borgmatic', - ) - - DEFAULT_ARCHIVE_NAME = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}' # noqa: FS003 REPO_ARCHIVE = (f'repo::{DEFAULT_ARCHIVE_NAME}',) diff --git a/tests/unit/borg/test_flags.py b/tests/unit/borg/test_flags.py index 38228e1d..d4feec64 100644 --- a/tests/unit/borg/test_flags.py +++ b/tests/unit/borg/test_flags.py @@ -285,3 +285,35 @@ def test_warn_for_aggressive_archive_flags_with_glob_archives_and_json_missing_a flexmock(module.logger).should_receive('warning').never() module.warn_for_aggressive_archive_flags(('borg', '--glob-archives', 'foo*'), '{}') + + +def test_omit_flag_removes_flag_from_arguments(): + module.omit_flag(('borg', 'create', '--flag', '--other'), '--flag') == ( + 'borg', + 'create', + '--other', + ) + + +def test_omit_flag_without_flag_present_passes_through_arguments(): + module.omit_flag(('borg', 'create', '--other'), '--flag') == ('borg', 'create', '--other') + + +def test_omit_flag_and_value_removes_flag_and_value_from_arguments(): + module.omit_flag(('borg', 'create', '--flag', 'value', '--other'), '--flag') == ( + 'borg', + 'create', + '--other', + ) + + +def test_omit_flag_and_value_with_equals_sign_removes_flag_and_value_from_arguments(): + module.omit_flag(('borg', 'create', '--flag=value', '--other'), '--flag') == ( + 'borg', + 'create', + '--other', + ) + + +def test_omit_flag_and_value_without_flag_present_passes_through_arguments(): + module.omit_flag(('borg', 'create', '--other'), '--flag') == ('borg', 'create', '--other') From 3642687ab507832ec33fa9660db74457fa4d874f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Feb 2025 14:32:32 -0800 Subject: [PATCH 045/226] Fix broken tests (#999). --- tests/unit/borg/test_flags.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/tests/unit/borg/test_flags.py b/tests/unit/borg/test_flags.py index d4feec64..63dbab35 100644 --- a/tests/unit/borg/test_flags.py +++ b/tests/unit/borg/test_flags.py @@ -288,7 +288,7 @@ def test_warn_for_aggressive_archive_flags_with_glob_archives_and_json_missing_a def test_omit_flag_removes_flag_from_arguments(): - module.omit_flag(('borg', 'create', '--flag', '--other'), '--flag') == ( + assert module.omit_flag(('borg', 'create', '--flag', '--other'), '--flag') == ( 'borg', 'create', '--other', @@ -296,11 +296,17 @@ def test_omit_flag_removes_flag_from_arguments(): def test_omit_flag_without_flag_present_passes_through_arguments(): - module.omit_flag(('borg', 'create', '--other'), '--flag') == ('borg', 'create', '--other') + assert module.omit_flag(('borg', 'create', '--other'), '--flag') == ( + 'borg', + 'create', + '--other', + ) def test_omit_flag_and_value_removes_flag_and_value_from_arguments(): - module.omit_flag(('borg', 'create', '--flag', 'value', '--other'), '--flag') == ( + assert module.omit_flag_and_value( + ('borg', 'create', '--flag', 'value', '--other'), '--flag' + ) == ( 'borg', 'create', '--other', @@ -308,7 +314,7 @@ def test_omit_flag_and_value_removes_flag_and_value_from_arguments(): def test_omit_flag_and_value_with_equals_sign_removes_flag_and_value_from_arguments(): - module.omit_flag(('borg', 'create', '--flag=value', '--other'), '--flag') == ( + assert module.omit_flag_and_value(('borg', 'create', '--flag=value', '--other'), '--flag') == ( 'borg', 'create', '--other', @@ -316,4 +322,8 @@ def test_omit_flag_and_value_with_equals_sign_removes_flag_and_value_from_argume def test_omit_flag_and_value_without_flag_present_passes_through_arguments(): - module.omit_flag(('borg', 'create', '--other'), '--flag') == ('borg', 'create', '--other') + assert module.omit_flag_and_value(('borg', 'create', '--other'), '--flag') == ( + 'borg', + 'create', + '--other', + ) From 4f88018558c285140660a9529b35acc9c20082f9 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Feb 2025 14:39:45 -0800 Subject: [PATCH 046/226] Bump version for release. --- NEWS | 2 +- borgmatic/actions/change_passphrase.py | 2 +- docs/how-to/provide-your-passwords.md | 2 +- pyproject.toml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index cc905bba..bd76c1be 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -1.9.11.dev0 +1.9.11 * #795: Add credential loading from file, KeePassXC, and Docker/Podman secrets. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/ diff --git a/borgmatic/actions/change_passphrase.py b/borgmatic/actions/change_passphrase.py index 7c4a3974..6d0ff31c 100644 --- a/borgmatic/actions/change_passphrase.py +++ b/borgmatic/actions/change_passphrase.py @@ -16,7 +16,7 @@ def run_change_passphrase( remote_path, ): ''' - Run the "key change-passprhase" action for the given repository. + Run the "key change-passphrase" action for the given repository. ''' if ( change_passphrase_arguments.repository is None diff --git a/docs/how-to/provide-your-passwords.md b/docs/how-to/provide-your-passwords.md index 4fae07b9..7ce3c8c3 100644 --- a/docs/how-to/provide-your-passwords.md +++ b/docs/how-to/provide-your-passwords.md @@ -35,7 +35,7 @@ encryption_passcommand: pass path/to/borg-passphrase New in version 1.9.9 Instead of letting Borg run the passcommand—potentially multiple times since borgmatic runs Borg multiple times—borgmatic now runs the passcommand itself and passes the -resulting passprhase securely to Borg via an anonymous pipe. This means you +resulting passphrase securely to Borg via an anonymous pipe. This means you should only ever get prompted for your password manager's passphrase at most once per borgmatic run. diff --git a/pyproject.toml b/pyproject.toml index d886f79c..61445bf0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "borgmatic" -version = "1.9.11.dev0" +version = "1.9.11" authors = [ { name="Dan Helfman", email="witten@torsion.org" }, ] From 4f0142c3c5b1edf15fd265d689f260dab026bbe5 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 23 Feb 2025 09:09:47 -0800 Subject: [PATCH 047/226] Fix Python < 3.12 compatibility issue (#1005). --- NEWS | 4 ++++ borgmatic/hooks/credential/container.py | 4 +++- borgmatic/hooks/credential/file.py | 4 +++- borgmatic/hooks/credential/systemd.py | 6 +++--- pyproject.toml | 2 +- 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/NEWS b/NEWS index bd76c1be..0ca65473 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,7 @@ +1.9.12.dev0 + * #1005: Fix the credential hooks to avoid using Python 3.12+ string features. Now borgmatic will + work with Python 3.9, 3.10, and 3.11 again. + 1.9.11 * #795: Add credential loading from file, KeePassXC, and Docker/Podman secrets. See the documentation for more information: diff --git a/borgmatic/hooks/credential/container.py b/borgmatic/hooks/credential/container.py index 5b41409b..a38e6d35 100644 --- a/borgmatic/hooks/credential/container.py +++ b/borgmatic/hooks/credential/container.py @@ -21,7 +21,9 @@ def load_credential(hook_config, config, credential_parameters): try: (secret_name,) = credential_parameters except ValueError: - raise ValueError(f'Cannot load invalid secret name: "{' '.join(credential_parameters)}"') + name = ' '.join(credential_parameters) + + raise ValueError(f'Cannot load invalid secret name: "{name}"') if not SECRET_NAME_PATTERN.match(secret_name): raise ValueError(f'Cannot load invalid secret name: "{secret_name}"') diff --git a/borgmatic/hooks/credential/file.py b/borgmatic/hooks/credential/file.py index 387b31b2..06854703 100644 --- a/borgmatic/hooks/credential/file.py +++ b/borgmatic/hooks/credential/file.py @@ -15,7 +15,9 @@ def load_credential(hook_config, config, credential_parameters): try: (credential_path,) = credential_parameters except ValueError: - raise ValueError(f'Cannot load invalid credential: "{' '.join(credential_parameters)}"') + name = ' '.join(credential_parameters) + + raise ValueError(f'Cannot load invalid credential: "{name}"') try: with open( diff --git a/borgmatic/hooks/credential/systemd.py b/borgmatic/hooks/credential/systemd.py index 085d9044..1de245a3 100644 --- a/borgmatic/hooks/credential/systemd.py +++ b/borgmatic/hooks/credential/systemd.py @@ -20,9 +20,9 @@ def load_credential(hook_config, config, credential_parameters): try: (credential_name,) = credential_parameters except ValueError: - raise ValueError( - f'Cannot load invalid credential name: "{' '.join(credential_parameters)}"' - ) + name = ' '.join(credential_parameters) + + raise ValueError(f'Cannot load invalid credential name: "{name}"') credentials_directory = os.environ.get('CREDENTIALS_DIRECTORY') diff --git a/pyproject.toml b/pyproject.toml index 61445bf0..d07f286a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "borgmatic" -version = "1.9.11" +version = "1.9.12.dev0" authors = [ { name="Dan Helfman", email="witten@torsion.org" }, ] From c462f0c84c8981494ffd0b1a7cb49c552acc7847 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 23 Feb 2025 09:59:19 -0800 Subject: [PATCH 048/226] Fix Python < 3.12 compatibility issue (#1005). --- borgmatic/hooks/credential/keepassxc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/borgmatic/hooks/credential/keepassxc.py b/borgmatic/hooks/credential/keepassxc.py index b280cc6a..e37be499 100644 --- a/borgmatic/hooks/credential/keepassxc.py +++ b/borgmatic/hooks/credential/keepassxc.py @@ -18,8 +18,10 @@ def load_credential(hook_config, config, credential_parameters): try: (database_path, attribute_name) = credential_parameters except ValueError: + path_and_name = ' '.join(credential_parameters) + raise ValueError( - f'Cannot load credential with invalid KeePassXC database path and attribute name: "{' '.join(credential_parameters)}"' + f'Cannot load credential with invalid KeePassXC database path and attribute name: "{path_and_name}"' ) if not os.path.exists(database_path): From 596305e3defe81b1e843d122357407d7ae0d382b Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 23 Feb 2025 09:59:53 -0800 Subject: [PATCH 049/226] Bump version for release. --- NEWS | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 0ca65473..c1203e4c 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -1.9.12.dev0 +1.9.12 * #1005: Fix the credential hooks to avoid using Python 3.12+ string features. Now borgmatic will work with Python 3.9, 3.10, and 3.11 again. diff --git a/pyproject.toml b/pyproject.toml index d07f286a..3d05a569 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "borgmatic" -version = "1.9.12.dev0" +version = "1.9.12" authors = [ { name="Dan Helfman", email="witten@torsion.org" }, ] From bcc463688a48225a120c5b24848199d794c6d9f5 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 23 Feb 2025 15:46:39 -0800 Subject: [PATCH 050/226] When getting all ZFS dataset mount points, deduplicate and filter out "none". --- NEWS | 3 +++ borgmatic/hooks/data_source/zfs.py | 13 +++++++++++-- pyproject.toml | 2 +- tests/unit/hooks/data_source/test_zfs.py | 20 ++++++++++++++++++-- 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/NEWS b/NEWS index c1203e4c..db012930 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +1.9.13.dev0 + * + 1.9.12 * #1005: Fix the credential hooks to avoid using Python 3.12+ string features. Now borgmatic will work with Python 3.9, 3.10, and 3.11 again. diff --git a/borgmatic/hooks/data_source/zfs.py b/borgmatic/hooks/data_source/zfs.py index a7e9f01d..f4d47bab 100644 --- a/borgmatic/hooks/data_source/zfs.py +++ b/borgmatic/hooks/data_source/zfs.py @@ -134,7 +134,16 @@ def get_all_dataset_mount_points(zfs_command): ) ) - return tuple(sorted(line.rstrip() for line in list_output.splitlines())) + return tuple( + sorted( + { + mount_point + for line in list_output.splitlines() + for mount_point in (line.rstrip(),) + if mount_point != 'none' + } + ) + ) def snapshot_dataset(zfs_command, full_snapshot_name): # pragma: no cover @@ -411,7 +420,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d continue if not dry_run: - shutil.rmtree(snapshots_directory) + shutil.rmtree(snapshots_directory, ignore_errors=True) # Destroy snapshots. full_snapshot_names = get_all_snapshots(zfs_command) diff --git a/pyproject.toml b/pyproject.toml index 3d05a569..42c64dc2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "borgmatic" -version = "1.9.12" +version = "1.9.13.dev0" authors = [ { name="Dan Helfman", email="witten@torsion.org" }, ] diff --git a/tests/unit/hooks/data_source/test_zfs.py b/tests/unit/hooks/data_source/test_zfs.py index fbe18fa9..74ca87b3 100644 --- a/tests/unit/hooks/data_source/test_zfs.py +++ b/tests/unit/hooks/data_source/test_zfs.py @@ -220,11 +220,27 @@ def test_get_datasets_to_backup_with_invalid_list_output_raises(): module.get_datasets_to_backup('zfs', patterns=(Pattern('/foo'), Pattern('/bar'))) -def test_get_all_dataset_mount_points_does_not_filter_datasets(): +def test_get_all_dataset_mount_points_omits_none(): flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).and_return( - '/dataset\n/other', + '/dataset\nnone\n/other', + ) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).and_return((Pattern('/dataset'),)) + + assert module.get_all_dataset_mount_points('zfs') == ( + ('/dataset'), + ('/other'), + ) + + +def test_get_all_dataset_mount_points_omits_duplicates(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( + '/dataset\n/other\n/dataset\n/other', ) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' From 8854b9ad20523148c657eb5771d8e2bceac31149 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 23 Feb 2025 15:49:12 -0800 Subject: [PATCH 051/226] Backing out a ZFS change that hasn't been confirmed working quite yet. --- borgmatic/hooks/data_source/zfs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borgmatic/hooks/data_source/zfs.py b/borgmatic/hooks/data_source/zfs.py index f4d47bab..2ca53c18 100644 --- a/borgmatic/hooks/data_source/zfs.py +++ b/borgmatic/hooks/data_source/zfs.py @@ -420,7 +420,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d continue if not dry_run: - shutil.rmtree(snapshots_directory, ignore_errors=True) + shutil.rmtree(snapshots_directory) # Destroy snapshots. full_snapshot_names = get_all_snapshots(zfs_command) From 3bca686707bcded37cd87e663bcb3a98caf61bf9 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 23 Feb 2025 17:01:35 -0800 Subject: [PATCH 052/226] Fix a ZFS error during snapshot cleanup (#1001). --- NEWS | 2 +- borgmatic/hooks/data_source/zfs.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index db012930..cc0d580f 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,5 @@ 1.9.13.dev0 - * + * #1001: Fix a ZFS error during snapshot cleanup. 1.9.12 * #1005: Fix the credential hooks to avoid using Python 3.12+ string features. Now borgmatic will diff --git a/borgmatic/hooks/data_source/zfs.py b/borgmatic/hooks/data_source/zfs.py index 2ca53c18..9cb77587 100644 --- a/borgmatic/hooks/data_source/zfs.py +++ b/borgmatic/hooks/data_source/zfs.py @@ -420,7 +420,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d continue if not dry_run: - shutil.rmtree(snapshots_directory) + shutil.rmtree(snapshot_mount_path, ignore_errors=True) # Destroy snapshots. full_snapshot_names = get_all_snapshots(zfs_command) From 2eab74a5212e947a5bc048d28678e9cdc2d7766b Mon Sep 17 00:00:00 2001 From: columbarius Date: Mon, 24 Feb 2025 19:26:57 +0100 Subject: [PATCH 053/226] Add "verify_tls" option to Uptime Kuma hook. --- borgmatic/config/schema.yaml | 6 +++ borgmatic/hooks/monitoring/uptime_kuma.py | 2 +- .../unit/hooks/monitoring/test_uptimekuma.py | 44 ++++++++++++++++--- 3 files changed, 45 insertions(+), 7 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 1e238721..8325e26e 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -2200,6 +2200,12 @@ properties: - start - finish - fail + verify_tls: + type: boolean + description: | + Verify the TLS certificate of the push URL host. Defaults to + true. + example: false description: | Configuration for a monitoring integration with Uptime Kuma using the Push monitor type. diff --git a/borgmatic/hooks/monitoring/uptime_kuma.py b/borgmatic/hooks/monitoring/uptime_kuma.py index 015dde0b..68d5f29b 100644 --- a/borgmatic/hooks/monitoring/uptime_kuma.py +++ b/borgmatic/hooks/monitoring/uptime_kuma.py @@ -37,7 +37,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev logging.getLogger('urllib3').setLevel(logging.ERROR) try: - response = requests.get(f'{push_url}?{query}') + response = requests.get(f'{push_url}?{query}', verify=hook_config.get('verify_tls', True)) if not response.ok: response.raise_for_status() except requests.exceptions.RequestException as error: diff --git a/tests/unit/hooks/monitoring/test_uptimekuma.py b/tests/unit/hooks/monitoring/test_uptimekuma.py index b29e1c44..a9d00bee 100644 --- a/tests/unit/hooks/monitoring/test_uptimekuma.py +++ b/tests/unit/hooks/monitoring/test_uptimekuma.py @@ -10,7 +10,7 @@ CUSTOM_PUSH_URL = 'https://uptime.example.com/api/push/efgh5678' def test_ping_monitor_hits_default_uptimekuma_on_fail(): hook_config = {} flexmock(module.requests).should_receive('get').with_args( - f'{DEFAULT_PUSH_URL}?status=down&msg=fail' + f'{DEFAULT_PUSH_URL}?status=down&msg=fail', verify=True ).and_return(flexmock(ok=True)).once() module.ping_monitor( @@ -26,7 +26,7 @@ def test_ping_monitor_hits_default_uptimekuma_on_fail(): def test_ping_monitor_hits_custom_uptimekuma_on_fail(): hook_config = {'push_url': CUSTOM_PUSH_URL} flexmock(module.requests).should_receive('get').with_args( - f'{CUSTOM_PUSH_URL}?status=down&msg=fail' + f'{CUSTOM_PUSH_URL}?status=down&msg=fail', verify=True ).and_return(flexmock(ok=True)).once() module.ping_monitor( @@ -42,7 +42,7 @@ def test_ping_monitor_hits_custom_uptimekuma_on_fail(): def test_ping_monitor_custom_uptimekuma_on_start(): hook_config = {'push_url': CUSTOM_PUSH_URL} flexmock(module.requests).should_receive('get').with_args( - f'{CUSTOM_PUSH_URL}?status=up&msg=start' + f'{CUSTOM_PUSH_URL}?status=up&msg=start', verify=True ).and_return(flexmock(ok=True)).once() module.ping_monitor( @@ -58,7 +58,7 @@ def test_ping_monitor_custom_uptimekuma_on_start(): def test_ping_monitor_custom_uptimekuma_on_finish(): hook_config = {'push_url': CUSTOM_PUSH_URL} flexmock(module.requests).should_receive('get').with_args( - f'{CUSTOM_PUSH_URL}?status=up&msg=finish' + f'{CUSTOM_PUSH_URL}?status=up&msg=finish', verify=True ).and_return(flexmock(ok=True)).once() module.ping_monitor( @@ -116,7 +116,7 @@ def test_ping_monitor_does_not_hit_custom_uptimekuma_on_finish_dry_run(): def test_ping_monitor_with_connection_error_logs_warning(): hook_config = {'push_url': CUSTOM_PUSH_URL} flexmock(module.requests).should_receive('get').with_args( - f'{CUSTOM_PUSH_URL}?status=down&msg=fail' + f'{CUSTOM_PUSH_URL}?status=down&msg=fail', verify=True ).and_raise(module.requests.exceptions.ConnectionError) flexmock(module.logger).should_receive('warning').once() @@ -137,7 +137,7 @@ def test_ping_monitor_with_other_error_logs_warning(): module.requests.exceptions.RequestException ) flexmock(module.requests).should_receive('get').with_args( - f'{CUSTOM_PUSH_URL}?status=down&msg=fail' + f'{CUSTOM_PUSH_URL}?status=down&msg=fail', verify=True ).and_return(response) flexmock(module.logger).should_receive('warning').once() @@ -163,3 +163,35 @@ def test_ping_monitor_with_invalid_run_state(): monitoring_log_level=1, dry_run=True, ) + + +def test_ping_monitor_skips_ssl_verification_when_verify_tls_false(): + hook_config = {'push_url': CUSTOM_PUSH_URL, 'verify_tls': False} + flexmock(module.requests).should_receive('get').with_args( + f'{CUSTOM_PUSH_URL}?status=down&msg=fail', verify=False + ).and_return(flexmock(ok=True)).once() + + module.ping_monitor( + hook_config, + {}, + 'config.yaml', + borgmatic.hooks.monitoring.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, + ) + + +def test_ping_monitor_executes_ssl_verification_when_verify_tls_true(): + hook_config = {'push_url': CUSTOM_PUSH_URL, 'verify_tls': True} + flexmock(module.requests).should_receive('get').with_args( + f'{CUSTOM_PUSH_URL}?status=down&msg=fail', verify=True + ).and_return(flexmock(ok=True)).once() + + module.ping_monitor( + hook_config, + {}, + 'config.yaml', + borgmatic.hooks.monitoring.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, + ) From 8a6c6c84d2050ef9cd15b768ed00f33cb0c47198 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 24 Feb 2025 11:30:16 -0800 Subject: [PATCH 054/226] Add Uptime Kuma "verify_tls" option to NEWS. --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index cc0d580f..d1f2f1fb 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ 1.9.13.dev0 * #1001: Fix a ZFS error during snapshot cleanup. + * Add a "verify_tls" option to the Uptime Kuma monitoring hook for disabling TLS verification. 1.9.12 * #1005: Fix the credential hooks to avoid using Python 3.12+ string features. Now borgmatic will From dce0528057c560f1239a89f0a39423c20c85b80c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 26 Feb 2025 22:33:01 -0800 Subject: [PATCH 055/226] In the Zabbix monitoring hook, support Zabbix 7.2's authentication changes (#1003). --- NEWS | 1 + borgmatic/config/schema.yaml | 5 +- borgmatic/hooks/monitoring/zabbix.py | 96 +++++--- docs/how-to/monitor-your-backups.md | 6 +- tests/unit/hooks/monitoring/test_zabbix.py | 271 +++++++++++++-------- 5 files changed, 245 insertions(+), 134 deletions(-) diff --git a/NEWS b/NEWS index d1f2f1fb..0d135f3b 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ 1.9.13.dev0 * #1001: Fix a ZFS error during snapshot cleanup. + * #1003: In the Zabbix monitoring hook, support Zabbix 7.2's authentication changes. * Add a "verify_tls" option to the Uptime Kuma monitoring hook for disabling TLS verification. 1.9.12 diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 8325e26e..33f39a70 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1907,6 +1907,8 @@ properties: zabbix: type: object additionalProperties: false + required: + - server properties: itemid: type: integer @@ -1929,7 +1931,8 @@ properties: server: type: string description: | - The address of your Zabbix instance. + The API endpoint URL of your Zabbix instance, usually ending + with "/api_jsonrpc.php". Required. example: https://zabbix.your-domain.com username: type: string diff --git a/borgmatic/hooks/monitoring/zabbix.py b/borgmatic/hooks/monitoring/zabbix.py index 54fcc196..9235147f 100644 --- a/borgmatic/hooks/monitoring/zabbix.py +++ b/borgmatic/hooks/monitoring/zabbix.py @@ -16,6 +16,42 @@ def initialize_monitor( pass +def send_zabbix_request(server, headers, data): + ''' + Given a Zabbix server URL, HTTP headers as a dict, and valid Zabbix JSON payload data as a dict, + send a request to the Zabbix server via API. + + Return the response "result" value or None. + ''' + logging.getLogger('urllib3').setLevel(logging.ERROR) + + logger.debug(f'Sending a "{data["method"]}" request to the Zabbix server') + + try: + response = requests.post(server, headers=headers, json=data) + + if not response.ok: + response.raise_for_status() + except requests.exceptions.RequestException as error: + logger.warning(f'Zabbix error: {error}') + + return None + + try: + result = response.json().get('result') + error_message = result['data'][0]['error'] + except requests.exceptions.JSONDecodeError: + logger.warning('Zabbix error: Cannot parse API response') + + return None + except (TypeError, KeyError, IndexError): + return result + else: + logger.warning(f'Zabbix error: {error_message}') + + return None + + def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run): ''' Update the configured Zabbix item using either the itemid, or a host and key. @@ -48,6 +84,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev ) except ValueError as error: logger.warning(f'Zabbix credential error: {error}') + return server = hook_config.get('server') @@ -57,13 +94,9 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev value = state_config.get('value') headers = {'Content-Type': 'application/json-rpc'} - logger.info(f'Updating Zabbix{dry_run_label}') + logger.info(f'Pinging Zabbix{dry_run_label}') logger.debug(f'Using Zabbix URL: {server}') - if server is None: - logger.warning('Server missing for Zabbix') - return - # Determine the Zabbix method used to store the value: itemid or host/key if itemid is not None: logger.info(f'Updating {itemid} on Zabbix') @@ -74,8 +107,8 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev 'id': 1, } - elif (host and key) is not None: - logger.info(f'Updating Host:{host} and Key:{key} on Zabbix') + elif host is not None and key is not None: + logger.info(f'Updating Host: "{host}" and Key: "{key}" on Zabbix') data = { 'jsonrpc': '2.0', 'method': 'history.push', @@ -85,58 +118,63 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev elif host is not None: logger.warning('Key missing for Zabbix') - return + return elif key is not None: logger.warning('Host missing for Zabbix') + return else: logger.warning('No Zabbix itemid or host/key provided') + return # Determine the authentication method: API key or username/password if api_key is not None: logger.info('Using API key auth for Zabbix') - headers['Authorization'] = 'Bearer ' + api_key - - elif (username and password) is not None: - logger.info('Using user/pass auth with user {username} for Zabbix') - auth_data = { + headers['Authorization'] = f'Bearer {api_key}' + elif username is not None and password is not None: + logger.info(f'Using user/pass auth with user {username} for Zabbix') + login_data = { 'jsonrpc': '2.0', 'method': 'user.login', 'params': {'username': username, 'password': password}, 'id': 1, } + if not dry_run: - logging.getLogger('urllib3').setLevel(logging.ERROR) - try: - response = requests.post(server, headers=headers, json=auth_data) - data['auth'] = response.json().get('result') - if not response.ok: - response.raise_for_status() - except requests.exceptions.RequestException as error: - logger.warning(f'Zabbix error: {error}') + result = send_zabbix_request(server, headers, login_data) + + if not result: return + headers['Authorization'] = f'Bearer {result}' elif username is not None: logger.warning('Password missing for Zabbix authentication') - return + return elif password is not None: logger.warning('Username missing for Zabbix authentication') + return else: logger.warning('Authentication data missing for Zabbix') + return if not dry_run: - logging.getLogger('urllib3').setLevel(logging.ERROR) - try: - response = requests.post(server, headers=headers, json=data) - if not response.ok: - response.raise_for_status() - except requests.exceptions.RequestException as error: - logger.warning(f'Zabbix error: {error}') + send_zabbix_request(server, headers, data) + + if username is not None and password is not None: + logout_data = { + 'jsonrpc': '2.0', + 'method': 'user.logout', + 'params': [], + 'id': 1, + } + + if not dry_run: + send_zabbix_request(server, headers, logout_data) def destroy_monitor(ping_url_or_uuid, config, monitoring_log_level, dry_run): # pragma: no cover diff --git a/docs/how-to/monitor-your-backups.md b/docs/how-to/monitor-your-backups.md index ff68f668..01649315 100644 --- a/docs/how-to/monitor-your-backups.md +++ b/docs/how-to/monitor-your-backups.md @@ -710,8 +710,10 @@ zabbix: - fail ``` -This hook requires the Zabbix server be running version 7.0. ([Support for Zabbix -7.2+](https://projects.torsion.org/borgmatic-collective/borgmatic/issues/1003) is planned.) +This hook requires the Zabbix server be running version 7.0. + +New in version 1.9.3 Zabbix 7.2+ +is supported as well. ### Authentication methods diff --git a/tests/unit/hooks/monitoring/test_zabbix.py b/tests/unit/hooks/monitoring/test_zabbix.py index 1b237810..0c2ea577 100644 --- a/tests/unit/hooks/monitoring/test_zabbix.py +++ b/tests/unit/hooks/monitoring/test_zabbix.py @@ -24,7 +24,6 @@ DATA_HOST_KEY_WITH_KEY_VALUE = { 'method': 'history.push', 'params': {'host': HOST, 'key': KEY, 'value': VALUE}, 'id': 1, - 'auth': '3fe6ed01a69ebd79907a120bcd04e494', } DATA_ITEMID = { @@ -39,7 +38,6 @@ DATA_HOST_KEY_WITH_ITEMID = { 'method': 'history.push', 'params': {'itemid': ITEMID, 'value': VALUE}, 'id': 1, - 'auth': '3fe6ed01a69ebd79907a120bcd04e494', } DATA_USER_LOGIN = { @@ -49,17 +47,99 @@ DATA_USER_LOGIN = { 'id': 1, } -AUTH_HEADERS_API_KEY = { +DATA_USER_LOGOUT = { + 'jsonrpc': '2.0', + 'method': 'user.logout', + 'params': [], + 'id': 1, +} + +AUTH_HEADERS_LOGIN = { + 'Content-Type': 'application/json-rpc', +} + +AUTH_HEADERS = { 'Content-Type': 'application/json-rpc', 'Authorization': f'Bearer {API_KEY}', } -AUTH_HEADERS_USERNAME_PASSWORD = {'Content-Type': 'application/json-rpc'} + +def test_send_zabbix_request_with_post_error_bails(): + server = flexmock() + headers = flexmock() + data = {'method': 'do.stuff'} + response = flexmock(ok=False) + response.should_receive('raise_for_status').and_raise( + module.requests.exceptions.RequestException + ) + + flexmock(module.requests).should_receive('post').with_args( + server, headers=headers, json=data + ).and_return(response) + + assert module.send_zabbix_request(server, headers, data) is None + + +def test_send_zabbix_request_with_invalid_json_response_bails(): + server = flexmock() + headers = flexmock() + data = {'method': 'do.stuff'} + flexmock(module.requests.exceptions.JSONDecodeError).should_receive('__init__') + response = flexmock(ok=True) + response.should_receive('json').and_raise(module.requests.exceptions.JSONDecodeError) + + flexmock(module.requests).should_receive('post').with_args( + server, headers=headers, json=data + ).and_return(response) + + assert module.send_zabbix_request(server, headers, data) is None + + +def test_send_zabbix_request_with_success_returns_response_result(): + server = flexmock() + headers = flexmock() + data = {'method': 'do.stuff'} + response = flexmock(ok=True) + response.should_receive('json').and_return({'result': {'foo': 'bar'}}) + + flexmock(module.requests).should_receive('post').with_args( + server, headers=headers, json=data + ).and_return(response) + + assert module.send_zabbix_request(server, headers, data) == {'foo': 'bar'} + + +def test_send_zabbix_request_with_success_passes_through_missing_result(): + server = flexmock() + headers = flexmock() + data = {'method': 'do.stuff'} + response = flexmock(ok=True) + response.should_receive('json').and_return({}) + + flexmock(module.requests).should_receive('post').with_args( + server, headers=headers, json=data + ).and_return(response) + + assert module.send_zabbix_request(server, headers, data) is None + + +def test_send_zabbix_request_with_error_bails(): + server = flexmock() + headers = flexmock() + data = {'method': 'do.stuff'} + response = flexmock(ok=True) + response.should_receive('json').and_return({'result': {'data': [{'error': 'oops'}]}}) + + flexmock(module.requests).should_receive('post').with_args( + server, headers=headers, json=data + ).and_return(response) + + assert module.send_zabbix_request(server, headers, data) is None def test_ping_monitor_with_non_matching_state_bails(): hook_config = {'api_key': API_KEY} - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() module.ping_monitor( hook_config, @@ -79,7 +159,7 @@ def test_ping_monitor_config_with_api_key_only_bails(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() module.ping_monitor( hook_config, @@ -99,7 +179,7 @@ def test_ping_monitor_config_with_host_only_bails(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() module.ping_monitor( hook_config, @@ -119,7 +199,7 @@ def test_ping_monitor_config_with_key_only_bails(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() module.ping_monitor( hook_config, @@ -139,7 +219,7 @@ def test_ping_monitor_config_with_server_only_bails(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() module.ping_monitor( hook_config, @@ -158,7 +238,7 @@ def test_ping_monitor_config_user_password_no_zabbix_data_bails(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() module.ping_monitor( hook_config, @@ -177,7 +257,7 @@ def test_ping_monitor_config_api_key_no_zabbix_data_bails(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() module.ping_monitor( hook_config, @@ -197,7 +277,7 @@ def test_ping_monitor_config_itemid_no_auth_data_bails(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() module.ping_monitor( hook_config, @@ -217,7 +297,7 @@ def test_ping_monitor_config_host_and_key_no_auth_data_bails(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() module.ping_monitor( hook_config, @@ -236,11 +316,35 @@ def test_ping_monitor_config_host_and_key_with_api_key_auth_data_successful(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module.requests).should_receive('post').with_args( + flexmock(module).should_receive('send_zabbix_request').with_args( f'{SERVER}', - headers=AUTH_HEADERS_API_KEY, - json=DATA_HOST_KEY, - ).and_return(flexmock(ok=True)).once() + headers=AUTH_HEADERS, + data=DATA_HOST_KEY, + ).once() + flexmock(module.logger).should_receive('warning').never() + + module.ping_monitor( + hook_config, + {}, + 'config.yaml', + borgmatic.hooks.monitoring.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, + ) + + +def test_ping_monitor_config_adds_missing_api_endpoint_to_server_url(): + # This test should simulate a successful POST to a Zabbix server. This test uses API_KEY + # to authenticate and HOST/KEY to know which item to populate in Zabbix. + hook_config = {'server': SERVER, 'host': HOST, 'key': KEY, 'api_key': API_KEY} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('send_zabbix_request').with_args( + f'{SERVER}', + headers=AUTH_HEADERS, + data=DATA_HOST_KEY, + ).once() flexmock(module.logger).should_receive('warning').never() module.ping_monitor( @@ -259,7 +363,7 @@ def test_ping_monitor_config_host_and_missing_key_bails(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() module.ping_monitor( hook_config, @@ -277,7 +381,7 @@ def test_ping_monitor_config_key_and_missing_host_bails(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() module.ping_monitor( hook_config, @@ -303,24 +407,26 @@ def test_ping_monitor_config_host_and_key_with_username_password_auth_data_succe flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - auth_response = flexmock(ok=True) - auth_response.should_receive('json').and_return( - {'jsonrpc': '2.0', 'result': '3fe6ed01a69ebd79907a120bcd04e494', 'id': 1} - ) - flexmock(module.requests).should_receive('post').with_args( + flexmock(module).should_receive('send_zabbix_request').with_args( f'{SERVER}', - headers=AUTH_HEADERS_USERNAME_PASSWORD, - json=DATA_USER_LOGIN, - ).and_return(auth_response).once() + headers=AUTH_HEADERS_LOGIN, + data=DATA_USER_LOGIN, + ).and_return('fakekey').once() flexmock(module.logger).should_receive('warning').never() - flexmock(module.requests).should_receive('post').with_args( + flexmock(module).should_receive('send_zabbix_request').with_args( f'{SERVER}', - headers=AUTH_HEADERS_USERNAME_PASSWORD, - json=DATA_HOST_KEY_WITH_KEY_VALUE, - ).and_return(flexmock(ok=True)).once() + headers=AUTH_HEADERS, + data=DATA_HOST_KEY_WITH_KEY_VALUE, + ).once() + + flexmock(module).should_receive('send_zabbix_request').with_args( + f'{SERVER}', + headers=AUTH_HEADERS, + data=DATA_USER_LOGOUT, + ).once() module.ping_monitor( hook_config, @@ -344,24 +450,22 @@ def test_ping_monitor_config_host_and_key_with_username_password_auth_data_and_a flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - auth_response = flexmock(ok=False) - auth_response.should_receive('json').and_return( - {'jsonrpc': '2.0', 'result': '3fe6ed01a69ebd79907a120bcd04e494', 'id': 1} - ) - auth_response.should_receive('raise_for_status').and_raise( - module.requests.ConnectionError - ).once() - flexmock(module.requests).should_receive('post').with_args( + flexmock(module).should_receive('send_zabbix_request').with_args( f'{SERVER}', - headers=AUTH_HEADERS_USERNAME_PASSWORD, - json=DATA_USER_LOGIN, - ).and_return(auth_response).once() - flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').with_args( + headers=AUTH_HEADERS_LOGIN, + data=DATA_USER_LOGIN, + ).and_return(None).once() + flexmock(module).should_receive('send_zabbix_request').with_args( f'{SERVER}', - headers=AUTH_HEADERS_USERNAME_PASSWORD, - json=DATA_HOST_KEY_WITH_KEY_VALUE, + headers=AUTH_HEADERS, + data=DATA_HOST_KEY_WITH_KEY_VALUE, + ).never() + + flexmock(module).should_receive('send_zabbix_request').with_args( + f'{SERVER}', + headers=AUTH_HEADERS, + data=DATA_USER_LOGOUT, ).never() module.ping_monitor( @@ -386,7 +490,7 @@ def test_ping_monitor_config_host_and_key_with_username_and_missing_password_bai 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() module.ping_monitor( hook_config, @@ -410,7 +514,7 @@ def test_ping_monitor_config_host_and_key_with_password_and_missing_username_bai 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() module.ping_monitor( hook_config, @@ -429,11 +533,11 @@ def test_ping_monitor_config_itemid_with_api_key_auth_data_successful(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module.requests).should_receive('post').with_args( + flexmock(module).should_receive('send_zabbix_request').with_args( f'{SERVER}', - headers=AUTH_HEADERS_API_KEY, - json=DATA_ITEMID, - ).and_return(flexmock(ok=True)).once() + headers=AUTH_HEADERS, + data=DATA_ITEMID, + ).once() flexmock(module.logger).should_receive('warning').never() module.ping_monitor( @@ -454,63 +558,26 @@ def test_ping_monitor_config_itemid_with_username_password_auth_data_successful( flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - auth_response = flexmock(ok=True) - auth_response.should_receive('json').and_return( - {'jsonrpc': '2.0', 'result': '3fe6ed01a69ebd79907a120bcd04e494', 'id': 1} - ) - flexmock(module.requests).should_receive('post').with_args( + flexmock(module).should_receive('send_zabbix_request').with_args( f'{SERVER}', - headers=AUTH_HEADERS_USERNAME_PASSWORD, - json=DATA_USER_LOGIN, - ).and_return(auth_response).once() + headers=AUTH_HEADERS_LOGIN, + data=DATA_USER_LOGIN, + ).and_return('fakekey').once() flexmock(module.logger).should_receive('warning').never() - flexmock(module.requests).should_receive('post').with_args( + flexmock(module).should_receive('send_zabbix_request').with_args( f'{SERVER}', - headers=AUTH_HEADERS_USERNAME_PASSWORD, - json=DATA_HOST_KEY_WITH_ITEMID, - ).and_return(flexmock(ok=True)).once() - - module.ping_monitor( - hook_config, - {}, - 'config.yaml', - borgmatic.hooks.monitoring.monitor.State.FAIL, - monitoring_log_level=1, - dry_run=False, - ) - - -def test_ping_monitor_config_itemid_with_username_password_auth_data_and_push_post_error_bails(): - hook_config = {'server': SERVER, 'itemid': ITEMID, 'username': USERNAME, 'password': PASSWORD} - - flexmock(module.borgmatic.hooks.credential.parse).should_receive( - 'resolve_credential' - ).replace_with(lambda value, config: value) - auth_response = flexmock(ok=True) - auth_response.should_receive('json').and_return( - {'jsonrpc': '2.0', 'result': '3fe6ed01a69ebd79907a120bcd04e494', 'id': 1} - ) - - flexmock(module.requests).should_receive('post').with_args( - f'{SERVER}', - headers=AUTH_HEADERS_USERNAME_PASSWORD, - json=DATA_USER_LOGIN, - ).and_return(auth_response).once() - - push_response = flexmock(ok=False) - push_response.should_receive('raise_for_status').and_raise( - module.requests.ConnectionError + headers=AUTH_HEADERS, + data=DATA_HOST_KEY_WITH_ITEMID, ).once() - flexmock(module.requests).should_receive('post').with_args( - f'{SERVER}', - headers=AUTH_HEADERS_USERNAME_PASSWORD, - json=DATA_HOST_KEY_WITH_ITEMID, - ).and_return(push_response).once() - flexmock(module.logger).should_receive('warning').once() + flexmock(module).should_receive('send_zabbix_request').with_args( + f'{SERVER}', + headers=AUTH_HEADERS, + data=DATA_USER_LOGOUT, + ).once() module.ping_monitor( hook_config, @@ -528,7 +595,7 @@ def test_ping_monitor_with_credential_error_bails(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).and_raise(ValueError) - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() flexmock(module.logger).should_receive('warning').once() module.ping_monitor( From 923fa7d82ff7303ef75beadf7b544aba091c2477 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 27 Feb 2025 09:23:08 -0800 Subject: [PATCH 056/226] Include contributors of closed tickets in "recent contributors" documentation. --- docs/fetch-contributors | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/fetch-contributors b/docs/fetch-contributors index 81ee6e22..7c1e1093 100755 --- a/docs/fetch-contributors +++ b/docs/fetch-contributors @@ -26,8 +26,7 @@ def list_merged_pulls(url): def list_contributing_issues(url): - # labels = bug, design finalized, etc. - response = requests.get(f'{url}?labels=19,20,22,23,32,52,53,54', headers={'Accept': 'application/json', 'Content-Type': 'application/json'}) + response = requests.get(url, headers={'Accept': 'application/json', 'Content-Type': 'application/json'}) if not response.ok: response.raise_for_status() @@ -39,7 +38,7 @@ PULLS_API_ENDPOINT_URLS = ( 'https://projects.torsion.org/api/v1/repos/borgmatic-collective/borgmatic/pulls', 'https://api.github.com/repos/borgmatic-collective/borgmatic/pulls', ) -ISSUES_API_ENDPOINT_URL = 'https://projects.torsion.org/api/v1/repos/borgmatic-collective/borgmatic/issues' +ISSUES_API_ENDPOINT_URL = 'https://projects.torsion.org/api/v1/repos/borgmatic-collective/borgmatic/issues?state=all' RECENT_CONTRIBUTORS_CUTOFF_DAYS = 365 From 0bd418836e01ac85ff5b60e4321e2d8881da20c0 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 27 Feb 2025 10:15:45 -0800 Subject: [PATCH 057/226] Send MariaDB passwords via anonymous pipe instead of environment variable (#1009) --- borgmatic/borg/environment.py | 2 +- borgmatic/execute.py | 35 +++++---- borgmatic/hooks/data_source/mariadb.py | 103 ++++++++++++++++--------- 3 files changed, 87 insertions(+), 53 deletions(-) diff --git a/borgmatic/borg/environment.py b/borgmatic/borg/environment.py index 19c8ad6e..344478c2 100644 --- a/borgmatic/borg/environment.py +++ b/borgmatic/borg/environment.py @@ -74,7 +74,7 @@ def make_environment(config): os.write(write_file_descriptor, passphrase.encode('utf-8')) os.close(write_file_descriptor) - # This, plus subprocess.Popen(..., close_fds=False) in execute.py, is necessary for the Borg + # This plus subprocess.Popen(..., close_fds=False) in execute.py is necessary for the Borg # child process to inherit the file descriptor. os.set_inheritable(read_file_descriptor, True) environment['BORG_PASSPHRASE_FD'] = str(read_file_descriptor) diff --git a/borgmatic/execute.py b/borgmatic/execute.py index df58c833..20eedb58 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -266,8 +266,8 @@ def log_command(full_command, input_file=None, output_file=None, environment=Non width=MAX_LOGGED_COMMAND_LENGTH, placeholder=' ...', ) - + (f" < {getattr(input_file, 'name', '')}" if input_file else '') - + (f" > {getattr(output_file, 'name', '')}" if output_file else '') + + (f" < {getattr(input_file, 'name', input_file)}" if input_file else '') + + (f" > {getattr(output_file, 'name', output_file)}" if output_file else '') ) @@ -315,8 +315,8 @@ def execute_command( shell=shell, env=environment, cwd=working_directory, - # Necessary for the passcommand credential hook to work. - close_fds=not bool((environment or {}).get('BORG_PASSPHRASE_FD')), + # Necessary for passing credentials via anonymous pipe. + close_fds=False, ) if not run_to_completion: return process @@ -333,6 +333,7 @@ def execute_command( def execute_command_and_capture_output( full_command, + input_file=None, capture_stderr=False, shell=False, environment=None, @@ -342,28 +343,30 @@ def execute_command_and_capture_output( ): ''' Execute the given command (a sequence of command/argument strings), capturing and returning its - output (stdout). If capture stderr is True, then capture and return stderr in addition to - stdout. If shell is True, execute the command within a shell. If an environment variables dict - is given, then pass it into the command. If a working directory is given, use that as the - present working directory when running the command. If a Borg local path is given, and the - command matches it (regardless of arguments), treat exit code 1 as a warning instead of an - error. But if Borg exit codes are given as a sequence of exit code configuration dicts, then use - that configuration to decide what's an error and what's a warning. + output (stdout). If an input file descriptor is given, then pipe it to the command's stdin. If + capture stderr is True, then capture and return stderr in addition to stdout. If shell is True, + execute the command within a shell. If an environment variables dict is given, then pass it into + the command. If a working directory is given, use that as the present working directory when + running the command. If a Borg local path is given, and the command matches it (regardless of + arguments), treat exit code 1 as a warning instead of an error. But if Borg exit codes are given + as a sequence of exit code configuration dicts, then use that configuration to decide what's an + error and what's a warning. Raise subprocesses.CalledProcessError if an error occurs while running the command. ''' - log_command(full_command, environment=environment) + log_command(full_command, input_file, environment=environment) command = ' '.join(full_command) if shell else full_command try: output = subprocess.check_output( command, + stdin=input_file, stderr=subprocess.STDOUT if capture_stderr else None, shell=shell, env=environment, cwd=working_directory, - # Necessary for the passcommand credential hook to work. - close_fds=not bool((environment or {}).get('BORG_PASSPHRASE_FD')), + # Necessary for passing credentials via anonymous pipe. + close_fds=False, ) except subprocess.CalledProcessError as error: if ( @@ -422,8 +425,8 @@ def execute_command_with_processes( shell=shell, env=environment, cwd=working_directory, - # Necessary for the passcommand credential hook to work. - close_fds=not bool((environment or {}).get('BORG_PASSPHRASE_FD')), + # Necessary for passing credentials via anonymous pipe. + close_fds=False, ) except (subprocess.CalledProcessError, OSError): # Something has gone wrong. So vent each process' output buffer to prevent it from hanging. diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index b992a944..b6be8b30 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -26,11 +26,56 @@ def make_dump_path(base_directory): # pragma: no cover SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys') -def database_names_to_dump(database, config, environment, dry_run): +def make_defaults_file_pipe(username=None, password=None): ''' - Given a requested database config and a configuration dict, return the corresponding sequence of - database names to dump. In the case of "all", query for the names of databases on the configured - host and return them, excluding any system databases that will cause problems during restore. + Given a database username and/or password, write it to an anonymous pipe and return its file + descriptor for passing to an executed command. The idea is that this is a more secure way to + transmit credentials to a database client than using an environment variable. + + If no username or password are given, then return None. + + Do not and use this value for multiple different command invocations. That will not work because + each pipe is "used up" once read. + ''' + values = '\n'.join( + ( + (f'user={username}' if username is not None else ''), + (f'password={password}' if password is not None else ''), + ) + ).strip() + + if not values: + return None + + fields_message = ' and '.join( + field_name for field_name in + ( + (f'username ({username})' if username is not None else None), + ('password' if password is not None else None), + ) + if field_name is not None + ) + logger.debug(f'Writing database {fields_message} to defaults extra file pipe') + + read_file_descriptor, write_file_descriptor = os.pipe() + os.write( + write_file_descriptor, f'[client]\n{values}'.encode('utf-8') + ) + os.close(write_file_descriptor) + + # This plus subprocess.Popen(..., close_fds=False) in execute.py is necessary for the database + # client child process to inherit the file descriptor. + os.set_inheritable(read_file_descriptor, True) + + return read_file_descriptor + + +def database_names_to_dump(database, config, username, password, environment, dry_run): + ''' + Given a requested database config, a configuration dict, a database username and password, an + environment dict, and whether this is a dry run, return the corresponding sequence of database + names to dump. In the case of "all", query for the names of databases on the configured host and + return them, excluding any system databases that will cause problems during restore. ''' if database['name'] != 'all': return (database['name'],) @@ -40,24 +85,20 @@ def database_names_to_dump(database, config, environment, dry_run): mariadb_show_command = tuple( shlex.quote(part) for part in shlex.split(database.get('mariadb_command') or 'mariadb') ) + defaults_file_descriptor = make_defaults_file_pipe(username, password) show_command = ( mariadb_show_command + + ((f'--defaults-extra-file=/dev/fd/{defaults_file_descriptor}',) if defaults_file_descriptor else ()) + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) - + ( - ( - '--user', - borgmatic.hooks.credential.parse.resolve_credential(database['username'], config), - ) - if 'username' in database - else () - ) + ('--skip-column-names', '--batch') + ('--execute', 'show schemas') ) + logger.debug('Querying for "all" MariaDB databases to dump') + show_output = execute_command_and_capture_output(show_command, environment=environment) return tuple( @@ -68,7 +109,7 @@ def database_names_to_dump(database, config, environment, dry_run): def execute_dump_command( - database, config, dump_path, database_names, environment, dry_run, dry_run_label + database, config, username, password, dump_path, database_names, environment, dry_run, dry_run_label ): ''' Kick off a dump for the given MariaDB database (provided as a configuration dict) to a named @@ -95,21 +136,15 @@ def execute_dump_command( shlex.quote(part) for part in shlex.split(database.get('mariadb_dump_command') or 'mariadb-dump') ) + defaults_file_descriptor = make_defaults_file_pipe(username, password) dump_command = ( mariadb_dump_command + + ((f'--defaults-extra-file=/dev/fd/{defaults_file_descriptor}',) if defaults_file_descriptor else ()) + (tuple(database['options'].split(' ')) if 'options' in database else ()) + (('--add-drop-database',) if database.get('add_drop_database', True) else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) - + ( - ( - '--user', - borgmatic.hooks.credential.parse.resolve_credential(database['username'], config), - ) - if 'username' in database - else () - ) + ('--databases',) + database_names + ('--result-file', dump_filename) @@ -165,19 +200,10 @@ def dump_data_sources( for database in databases: dump_path = make_dump_path(borgmatic_runtime_directory) - environment = dict( - os.environ, - **( - { - 'MYSQL_PWD': borgmatic.hooks.credential.parse.resolve_credential( - database['password'], config - ) - } - if 'password' in database - else {} - ), - ) - dump_database_names = database_names_to_dump(database, config, environment, dry_run) + username = borgmatic.hooks.credential.parse.resolve_credential(database.get('username'), config) + password = borgmatic.hooks.credential.parse.resolve_credential(database.get('password'), config) + environment = dict(os.environ) + dump_database_names = database_names_to_dump(database, config, username, password, environment, dry_run) if not dump_database_names: if dry_run: @@ -193,6 +219,8 @@ def dump_data_sources( execute_dump_command( renamed_database, config, + username, + password, dump_path, (dump_name,), environment, @@ -205,6 +233,8 @@ def dump_data_sources( execute_dump_command( database, config, + username, + password, dump_path, dump_database_names, environment, @@ -296,8 +326,10 @@ def restore_data_source_dump( mariadb_restore_command = tuple( shlex.quote(part) for part in shlex.split(data_source.get('mariadb_command') or 'mariadb') ) + defaults_file_descriptor = make_defaults_file_pipe(username, password) restore_command = ( mariadb_restore_command + + ((f'--defaults-extra-file=/dev/fd/{defaults_file_descriptor}',) if defaults_file_descriptor else ()) + ('--batch',) + ( tuple(data_source['restore_options'].split(' ')) @@ -307,9 +339,8 @@ def restore_data_source_dump( + (('--host', hostname) if hostname else ()) + (('--port', str(port)) if port else ()) + (('--protocol', 'tcp') if hostname or port else ()) - + (('--user', username) if username else ()) ) - environment = dict(os.environ, **({'MYSQL_PWD': password} if password else {})) + environment = dict(os.environ) logger.debug(f"Restoring MariaDB database {data_source['name']}{dry_run_label}") if dry_run: From 36d00733757b7a9123332ddf22fc4429ba9c7587 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 27 Feb 2025 10:42:47 -0800 Subject: [PATCH 058/226] Send MySQL passwords via anonymous pipe instead of environment variable (#1009). --- borgmatic/hooks/data_source/mariadb.py | 2 +- borgmatic/hooks/data_source/mysql.py | 60 +++++++++++--------------- 2 files changed, 25 insertions(+), 37 deletions(-) diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index b6be8b30..572b147f 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -34,7 +34,7 @@ def make_defaults_file_pipe(username=None, password=None): If no username or password are given, then return None. - Do not and use this value for multiple different command invocations. That will not work because + Do not use this value for multiple different command invocations. That will not work because each pipe is "used up" once read. ''' values = '\n'.join( diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index 3172edbb..23175bd6 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -6,6 +6,7 @@ import shlex import borgmatic.borg.pattern import borgmatic.config.paths import borgmatic.hooks.credential.parse +import borgmatic.hooks.data_source.mariadb from borgmatic.execute import ( execute_command, execute_command_and_capture_output, @@ -26,11 +27,12 @@ def make_dump_path(base_directory): # pragma: no cover SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys') -def database_names_to_dump(database, config, environment, dry_run): +def database_names_to_dump(database, config, username, password, environment, dry_run): ''' - Given a requested database config and a configuration dict, return the corresponding sequence of - database names to dump. In the case of "all", query for the names of databases on the configured - host and return them, excluding any system databases that will cause problems during restore. + Given a requested database config, a configuration dict, a database username and password, an + environment dict, and whether this is a dry run, return the corresponding sequence of database + names to dump. In the case of "all", query for the names of databases on the configured host and + return them, excluding any system databases that will cause problems during restore. ''' if database['name'] != 'all': return (database['name'],) @@ -40,24 +42,20 @@ def database_names_to_dump(database, config, environment, dry_run): mysql_show_command = tuple( shlex.quote(part) for part in shlex.split(database.get('mysql_command') or 'mysql') ) + defaults_file_descriptor = borgmatic.hooks.data_source.mariadb.make_defaults_file_pipe(username, password) show_command = ( mysql_show_command + + ((f'--defaults-extra-file=/dev/fd/{defaults_file_descriptor}',) if defaults_file_descriptor else ()) + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) - + ( - ( - '--user', - borgmatic.hooks.credential.parse.resolve_credential(database['username'], config), - ) - if 'username' in database - else () - ) + ('--skip-column-names', '--batch') + ('--execute', 'show schemas') ) + logger.debug('Querying for "all" MySQL databases to dump') + show_output = execute_command_and_capture_output(show_command, environment=environment) return tuple( @@ -68,7 +66,7 @@ def database_names_to_dump(database, config, environment, dry_run): def execute_dump_command( - database, config, dump_path, database_names, environment, dry_run, dry_run_label + database, config, username, password, dump_path, database_names, environment, dry_run, dry_run_label ): ''' Kick off a dump for the given MySQL/MariaDB database (provided as a configuration dict) to a @@ -94,21 +92,15 @@ def execute_dump_command( mysql_dump_command = tuple( shlex.quote(part) for part in shlex.split(database.get('mysql_dump_command') or 'mysqldump') ) + defaults_file_descriptor = borgmatic.hooks.data_source.mariadb.make_defaults_file_pipe(username, password) dump_command = ( mysql_dump_command + + ((f'--defaults-extra-file=/dev/fd/{defaults_file_descriptor}',) if defaults_file_descriptor else ()) + (tuple(database['options'].split(' ')) if 'options' in database else ()) + (('--add-drop-database',) if database.get('add_drop_database', True) else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) - + ( - ( - '--user', - borgmatic.hooks.credential.parse.resolve_credential(database['username'], config), - ) - if 'username' in database - else () - ) + ('--databases',) + database_names + ('--result-file', dump_filename) @@ -164,19 +156,10 @@ def dump_data_sources( for database in databases: dump_path = make_dump_path(borgmatic_runtime_directory) - environment = dict( - os.environ, - **( - { - 'MYSQL_PWD': borgmatic.hooks.credential.parse.resolve_credential( - database['password'], config - ) - } - if 'password' in database - else {} - ), - ) - dump_database_names = database_names_to_dump(database, config, environment, dry_run) + username = borgmatic.hooks.credential.parse.resolve_credential(database.get('username'), config) + password = borgmatic.hooks.credential.parse.resolve_credential(database.get('password'), config) + environment = dict(os.environ) + dump_database_names = database_names_to_dump(database, config, username, password, environment, dry_run) if not dump_database_names: if dry_run: @@ -192,6 +175,8 @@ def dump_data_sources( execute_dump_command( renamed_database, config, + username, + password, dump_path, (dump_name,), environment, @@ -204,6 +189,8 @@ def dump_data_sources( execute_dump_command( database, config, + username, + password, dump_path, dump_database_names, environment, @@ -295,8 +282,10 @@ def restore_data_source_dump( mysql_restore_command = tuple( shlex.quote(part) for part in shlex.split(data_source.get('mysql_command') or 'mysql') ) + defaults_file_descriptor = borgmatic.hooks.data_source.mariadb.make_defaults_file_pipe(username, password) restore_command = ( mysql_restore_command + + ((f'--defaults-extra-file=/dev/fd/{defaults_file_descriptor}',) if defaults_file_descriptor else ()) + ('--batch',) + ( tuple(data_source['restore_options'].split(' ')) @@ -306,9 +295,8 @@ def restore_data_source_dump( + (('--host', hostname) if hostname else ()) + (('--port', str(port)) if port else ()) + (('--protocol', 'tcp') if hostname or port else ()) - + (('--user', username) if username else ()) ) - environment = dict(os.environ, **({'MYSQL_PWD': password} if password else {})) + environment = dict(os.environ) logger.debug(f"Restoring MySQL database {data_source['name']}{dry_run_label}") if dry_run: From c41b743819b846b44e9c8dedf7cee8208428d414 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 28 Feb 2025 08:37:03 -0800 Subject: [PATCH 059/226] Get existing unit tests passing (#1009). --- borgmatic/hooks/data_source/mariadb.py | 48 ++++-- borgmatic/hooks/data_source/mysql.py | 52 +++++-- tests/unit/hooks/data_source/test_mariadb.py | 146 +++++++++++++++---- tests/unit/hooks/data_source/test_mysql.py | 146 ++++++++++++++++--- tests/unit/test_execute.py | 124 ++++------------ 5 files changed, 350 insertions(+), 166 deletions(-) diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index 572b147f..c57abf7a 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -48,8 +48,8 @@ def make_defaults_file_pipe(username=None, password=None): return None fields_message = ' and '.join( - field_name for field_name in - ( + field_name + for field_name in ( (f'username ({username})' if username is not None else None), ('password' if password is not None else None), ) @@ -58,9 +58,7 @@ def make_defaults_file_pipe(username=None, password=None): logger.debug(f'Writing database {fields_message} to defaults extra file pipe') read_file_descriptor, write_file_descriptor = os.pipe() - os.write( - write_file_descriptor, f'[client]\n{values}'.encode('utf-8') - ) + os.write(write_file_descriptor, f'[client]\n{values}'.encode('utf-8')) os.close(write_file_descriptor) # This plus subprocess.Popen(..., close_fds=False) in execute.py is necessary for the database @@ -88,7 +86,11 @@ def database_names_to_dump(database, config, username, password, environment, dr defaults_file_descriptor = make_defaults_file_pipe(username, password) show_command = ( mariadb_show_command - + ((f'--defaults-extra-file=/dev/fd/{defaults_file_descriptor}',) if defaults_file_descriptor else ()) + + ( + (f'--defaults-extra-file=/dev/fd/{defaults_file_descriptor}',) + if defaults_file_descriptor + else () + ) + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) @@ -109,7 +111,15 @@ def database_names_to_dump(database, config, username, password, environment, dr def execute_dump_command( - database, config, username, password, dump_path, database_names, environment, dry_run, dry_run_label + database, + config, + username, + password, + dump_path, + database_names, + environment, + dry_run, + dry_run_label, ): ''' Kick off a dump for the given MariaDB database (provided as a configuration dict) to a named @@ -139,7 +149,11 @@ def execute_dump_command( defaults_file_descriptor = make_defaults_file_pipe(username, password) dump_command = ( mariadb_dump_command - + ((f'--defaults-extra-file=/dev/fd/{defaults_file_descriptor}',) if defaults_file_descriptor else ()) + + ( + (f'--defaults-extra-file=/dev/fd/{defaults_file_descriptor}',) + if defaults_file_descriptor + else () + ) + (tuple(database['options'].split(' ')) if 'options' in database else ()) + (('--add-drop-database',) if database.get('add_drop_database', True) else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) @@ -200,10 +214,16 @@ def dump_data_sources( for database in databases: dump_path = make_dump_path(borgmatic_runtime_directory) - username = borgmatic.hooks.credential.parse.resolve_credential(database.get('username'), config) - password = borgmatic.hooks.credential.parse.resolve_credential(database.get('password'), config) + username = borgmatic.hooks.credential.parse.resolve_credential( + database.get('username'), config + ) + password = borgmatic.hooks.credential.parse.resolve_credential( + database.get('password'), config + ) environment = dict(os.environ) - dump_database_names = database_names_to_dump(database, config, username, password, environment, dry_run) + dump_database_names = database_names_to_dump( + database, config, username, password, environment, dry_run + ) if not dump_database_names: if dry_run: @@ -329,7 +349,11 @@ def restore_data_source_dump( defaults_file_descriptor = make_defaults_file_pipe(username, password) restore_command = ( mariadb_restore_command - + ((f'--defaults-extra-file=/dev/fd/{defaults_file_descriptor}',) if defaults_file_descriptor else ()) + + ( + (f'--defaults-extra-file=/dev/fd/{defaults_file_descriptor}',) + if defaults_file_descriptor + else () + ) + ('--batch',) + ( tuple(data_source['restore_options'].split(' ')) diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index 23175bd6..a31f7a54 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -42,10 +42,16 @@ def database_names_to_dump(database, config, username, password, environment, dr mysql_show_command = tuple( shlex.quote(part) for part in shlex.split(database.get('mysql_command') or 'mysql') ) - defaults_file_descriptor = borgmatic.hooks.data_source.mariadb.make_defaults_file_pipe(username, password) + defaults_file_descriptor = borgmatic.hooks.data_source.mariadb.make_defaults_file_pipe( + username, password + ) show_command = ( mysql_show_command - + ((f'--defaults-extra-file=/dev/fd/{defaults_file_descriptor}',) if defaults_file_descriptor else ()) + + ( + (f'--defaults-extra-file=/dev/fd/{defaults_file_descriptor}',) + if defaults_file_descriptor + else () + ) + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) @@ -66,7 +72,15 @@ def database_names_to_dump(database, config, username, password, environment, dr def execute_dump_command( - database, config, username, password, dump_path, database_names, environment, dry_run, dry_run_label + database, + config, + username, + password, + dump_path, + database_names, + environment, + dry_run, + dry_run_label, ): ''' Kick off a dump for the given MySQL/MariaDB database (provided as a configuration dict) to a @@ -92,10 +106,16 @@ def execute_dump_command( mysql_dump_command = tuple( shlex.quote(part) for part in shlex.split(database.get('mysql_dump_command') or 'mysqldump') ) - defaults_file_descriptor = borgmatic.hooks.data_source.mariadb.make_defaults_file_pipe(username, password) + defaults_file_descriptor = borgmatic.hooks.data_source.mariadb.make_defaults_file_pipe( + username, password + ) dump_command = ( mysql_dump_command - + ((f'--defaults-extra-file=/dev/fd/{defaults_file_descriptor}',) if defaults_file_descriptor else ()) + + ( + (f'--defaults-extra-file=/dev/fd/{defaults_file_descriptor}',) + if defaults_file_descriptor + else () + ) + (tuple(database['options'].split(' ')) if 'options' in database else ()) + (('--add-drop-database',) if database.get('add_drop_database', True) else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) @@ -156,10 +176,16 @@ def dump_data_sources( for database in databases: dump_path = make_dump_path(borgmatic_runtime_directory) - username = borgmatic.hooks.credential.parse.resolve_credential(database.get('username'), config) - password = borgmatic.hooks.credential.parse.resolve_credential(database.get('password'), config) + username = borgmatic.hooks.credential.parse.resolve_credential( + database.get('username'), config + ) + password = borgmatic.hooks.credential.parse.resolve_credential( + database.get('password'), config + ) environment = dict(os.environ) - dump_database_names = database_names_to_dump(database, config, username, password, environment, dry_run) + dump_database_names = database_names_to_dump( + database, config, username, password, environment, dry_run + ) if not dump_database_names: if dry_run: @@ -282,10 +308,16 @@ def restore_data_source_dump( mysql_restore_command = tuple( shlex.quote(part) for part in shlex.split(data_source.get('mysql_command') or 'mysql') ) - defaults_file_descriptor = borgmatic.hooks.data_source.mariadb.make_defaults_file_pipe(username, password) + defaults_file_descriptor = borgmatic.hooks.data_source.mariadb.make_defaults_file_pipe( + username, password + ) restore_command = ( mysql_restore_command - + ((f'--defaults-extra-file=/dev/fd/{defaults_file_descriptor}',) if defaults_file_descriptor else ()) + + ( + (f'--defaults-extra-file=/dev/fd/{defaults_file_descriptor}',) + if defaults_file_descriptor + else () + ) + ('--batch',) + ( tuple(data_source['restore_options'].split(' ')) diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py index 474c7d6d..6db363b8 100644 --- a/tests/unit/hooks/data_source/test_mariadb.py +++ b/tests/unit/hooks/data_source/test_mariadb.py @@ -9,7 +9,9 @@ from borgmatic.hooks.data_source import mariadb as module def test_database_names_to_dump_passes_through_name(): environment = flexmock() - names = module.database_names_to_dump({'name': 'foo'}, {}, environment, dry_run=False) + names = module.database_names_to_dump( + {'name': 'foo'}, {}, 'root', 'trustsome1', environment, dry_run=False + ) assert names == ('foo',) @@ -18,7 +20,9 @@ def test_database_names_to_dump_bails_for_dry_run(): environment = flexmock() flexmock(module).should_receive('execute_command_and_capture_output').never() - names = module.database_names_to_dump({'name': 'all'}, {}, environment, dry_run=True) + names = module.database_names_to_dump( + {'name': 'all'}, {}, 'root', 'trustsome1', environment, dry_run=True + ) assert names == () @@ -28,12 +32,24 @@ def test_database_names_to_dump_queries_mariadb_for_database_names(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_defaults_file_pipe').with_args( + 'root', 'trustsome1' + ).and_return(99) flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('mariadb', '--skip-column-names', '--batch', '--execute', 'show schemas'), + ( + 'mariadb', + '--defaults-extra-file=/dev/fd/99', + '--skip-column-names', + '--batch', + '--execute', + 'show schemas', + ), environment=environment, ).and_return('foo\nbar\nmysql\n').once() - names = module.database_names_to_dump({'name': 'all'}, {}, environment, dry_run=False) + names = module.database_names_to_dump( + {'name': 'all'}, {}, 'root', 'trustsome1', environment, dry_run=False + ) assert names == ('foo', 'bar') @@ -53,6 +69,9 @@ def test_dump_data_sources_dumps_each_database(): databases = [{'name': 'foo'}, {'name': 'bar'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).and_return(None) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' @@ -65,6 +84,8 @@ def test_dump_data_sources_dumps_each_database(): flexmock(module).should_receive('execute_dump_command').with_args( database={'name': name}, config={}, + username=None, + password=None, dump_path=object, database_names=(name,), environment={'USER': 'root'}, @@ -89,10 +110,10 @@ def test_dump_data_sources_dumps_with_password(): database = {'name': 'foo', 'username': 'root', 'password': 'trustsome1'} process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( ('bar',) ) @@ -100,9 +121,11 @@ def test_dump_data_sources_dumps_with_password(): flexmock(module).should_receive('execute_dump_command').with_args( database=database, config={}, + username='root', + password='trustsome1', dump_path=object, database_names=('foo',), - environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'}, + environment={'USER': 'root'}, dry_run=object, dry_run_label=object, ).and_return(process).once() @@ -121,14 +144,16 @@ def test_dump_data_sources_dumps_all_databases_at_once(): databases = [{'name': 'all'}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) flexmock(module).should_receive('execute_dump_command').with_args( database={'name': 'all'}, config={}, + username=None, + password=None, dump_path=object, database_names=('foo', 'bar'), environment={'USER': 'root'}, @@ -150,16 +175,18 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured databases = [{'name': 'all', 'format': 'sql'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' - ).replace_with(lambda value, config: value) + ).and_return(None) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) for name, process in zip(('foo', 'bar'), processes): flexmock(module).should_receive('execute_dump_command').with_args( database={'name': name, 'format': 'sql'}, config={}, + username=None, + password=None, dump_path=object, database_names=(name,), environment={'USER': 'root'}, @@ -181,11 +208,15 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured def test_database_names_to_dump_runs_mariadb_with_list_options(): - database = {'name': 'all', 'list_options': '--defaults-extra-file=mariadb.cnf'} + database = {'name': 'all', 'list_options': '--defaults-file=mariadb.cnf'} + flexmock(module).should_receive('make_defaults_file_pipe').with_args( + 'root', 'trustsome1' + ).and_return(99) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ( 'mariadb', - '--defaults-extra-file=mariadb.cnf', + '--defaults-extra-file=/dev/fd/99', + '--defaults-file=mariadb.cnf', '--skip-column-names', '--batch', '--execute', @@ -194,20 +225,27 @@ def test_database_names_to_dump_runs_mariadb_with_list_options(): environment=None, ).and_return(('foo\nbar')).once() - assert module.database_names_to_dump(database, {}, None, '') == ('foo', 'bar') + assert module.database_names_to_dump(database, {}, 'root', 'trustsome1', None, '') == ( + 'foo', + 'bar', + ) def test_database_names_to_dump_runs_non_default_mariadb_with_list_options(): database = { 'name': 'all', - 'list_options': '--defaults-extra-file=mariadb.cnf', + 'list_options': '--defaults-file=mariadb.cnf', 'mariadb_command': 'custom_mariadb', } + flexmock(module).should_receive('make_defaults_file_pipe').with_args( + 'root', 'trustsome1' + ).and_return(99) flexmock(module).should_receive('execute_command_and_capture_output').with_args( environment=None, full_command=( 'custom_mariadb', # Custom MariaDB command - '--defaults-extra-file=mariadb.cnf', + '--defaults-extra-file=/dev/fd/99', + '--defaults-file=mariadb.cnf', '--skip-column-names', '--batch', '--execute', @@ -215,7 +253,10 @@ def test_database_names_to_dump_runs_non_default_mariadb_with_list_options(): ), ).and_return(('foo\nbar')).once() - assert module.database_names_to_dump(database, {}, None, '') == ('foo', 'bar') + assert module.database_names_to_dump(database, {}, 'root', 'trustsome1', None, '') == ( + 'foo', + 'bar', + ) def test_execute_dump_command_runs_mariadb_dump(): @@ -225,11 +266,15 @@ def test_execute_dump_command_runs_mariadb_dump(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_defaults_file_pipe').with_args( + 'root', 'trustsome1' + ).and_return(99) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mariadb-dump', + '--defaults-extra-file=/dev/fd/99', '--add-drop-database', '--databases', 'foo', @@ -244,6 +289,8 @@ def test_execute_dump_command_runs_mariadb_dump(): module.execute_dump_command( database={'name': 'foo'}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -261,11 +308,15 @@ def test_execute_dump_command_runs_mariadb_dump_without_add_drop_database(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_defaults_file_pipe').with_args( + 'root', 'trustsome1' + ).and_return(99) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mariadb-dump', + '--defaults-extra-file=/dev/fd/99', '--databases', 'foo', '--result-file', @@ -279,6 +330,8 @@ def test_execute_dump_command_runs_mariadb_dump_without_add_drop_database(): module.execute_dump_command( database={'name': 'foo', 'add_drop_database': False}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -296,11 +349,15 @@ def test_execute_dump_command_runs_mariadb_dump_with_hostname_and_port(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_defaults_file_pipe').with_args( + 'root', 'trustsome1' + ).and_return(99) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mariadb-dump', + '--defaults-extra-file=/dev/fd/99', '--add-drop-database', '--host', 'database.example.org', @@ -321,6 +378,8 @@ def test_execute_dump_command_runs_mariadb_dump_with_hostname_and_port(): module.execute_dump_command( database={'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -338,20 +397,22 @@ def test_execute_dump_command_runs_mariadb_dump_with_username_and_password(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_defaults_file_pipe').with_args( + 'root', 'trustsome1' + ).and_return(99) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mariadb-dump', + '--defaults-extra-file=/dev/fd/99', '--add-drop-database', - '--user', - 'root', '--databases', 'foo', '--result-file', 'dump', ), - environment={'MYSQL_PWD': 'trustsome1'}, + environment={}, run_to_completion=False, ).and_return(process).once() @@ -359,9 +420,11 @@ def test_execute_dump_command_runs_mariadb_dump_with_username_and_password(): module.execute_dump_command( database={'name': 'foo', 'username': 'root', 'password': 'trustsome1'}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), - environment={'MYSQL_PWD': 'trustsome1'}, + environment={}, dry_run=False, dry_run_label='', ) @@ -376,11 +439,15 @@ def test_execute_dump_command_runs_mariadb_dump_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_defaults_file_pipe').with_args( + 'root', 'trustsome1' + ).and_return(99) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mariadb-dump', + '--defaults-extra-file=/dev/fd/99', '--stuff=such', '--add-drop-database', '--databases', @@ -396,6 +463,8 @@ def test_execute_dump_command_runs_mariadb_dump_with_options(): module.execute_dump_command( database={'name': 'foo', 'options': '--stuff=such'}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -413,11 +482,15 @@ def test_execute_dump_command_runs_non_default_mariadb_dump_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_defaults_file_pipe').with_args( + 'root', 'trustsome1' + ).and_return(99) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'custom_mariadb_dump', # Custom MariaDB dump command + '--defaults-extra-file=/dev/fd/99', '--stuff=such', '--add-drop-database', '--databases', @@ -437,6 +510,8 @@ def test_execute_dump_command_runs_non_default_mariadb_dump_with_options(): 'options': '--stuff=such', }, # Custom MariaDB dump command specified config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -450,6 +525,9 @@ def test_execute_dump_command_runs_non_default_mariadb_dump_with_options(): def test_execute_dump_command_with_duplicate_dump_skips_mariadb_dump(): flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module).should_receive('make_defaults_file_pipe').with_args( + 'root', 'trustsome1' + ).and_return(99) flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() @@ -457,6 +535,8 @@ def test_execute_dump_command_with_duplicate_dump_skips_mariadb_dump(): module.execute_dump_command( database={'name': 'foo'}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -473,6 +553,9 @@ def test_execute_dump_command_with_dry_run_skips_mariadb_dump(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_defaults_file_pipe').with_args( + 'root', 'trustsome1' + ).and_return(99) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').never() @@ -481,6 +564,8 @@ def test_execute_dump_command_with_dry_run_skips_mariadb_dump(): module.execute_dump_command( database={'name': 'foo'}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -685,13 +770,16 @@ def test_restore_data_source_dump_runs_mariadb_with_username_and_password(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_defaults_file_pipe').with_args( + 'root', 'trustsome1' + ).and_return(99) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( - ('mariadb', '--batch', '--user', 'root'), + ('mariadb', '--defaults-extra-file=/dev/fd/99', '--batch'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'}, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -727,10 +815,14 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_defaults_file_pipe').with_args( + 'cliusername', 'clipassword' + ).and_return(99) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mariadb', + '--defaults-extra-file=/dev/fd/99', '--batch', '--host', 'clihost', @@ -738,13 +830,11 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ 'cliport', '--protocol', 'tcp', - '--user', - 'cliusername', ), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - environment={'USER': 'root', 'MYSQL_PWD': 'clipassword'}, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -782,10 +872,14 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_defaults_file_pipe').with_args( + 'restoreuser', 'restorepass' + ).and_return(99) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mariadb', + '--defaults-extra-file=/dev/fd/99', '--batch', '--host', 'restorehost', @@ -793,13 +887,11 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ 'restoreport', '--protocol', 'tcp', - '--user', - 'restoreuser', ), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - environment={'USER': 'root', 'MYSQL_PWD': 'restorepass'}, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( diff --git a/tests/unit/hooks/data_source/test_mysql.py b/tests/unit/hooks/data_source/test_mysql.py index 5b4cbcc9..0fb3f432 100644 --- a/tests/unit/hooks/data_source/test_mysql.py +++ b/tests/unit/hooks/data_source/test_mysql.py @@ -9,7 +9,9 @@ from borgmatic.hooks.data_source import mysql as module def test_database_names_to_dump_passes_through_name(): environment = flexmock() - names = module.database_names_to_dump({'name': 'foo'}, {}, environment, dry_run=False) + names = module.database_names_to_dump( + {'name': 'foo'}, {}, 'root', 'trustsome1', environment, dry_run=False + ) assert names == ('foo',) @@ -21,7 +23,9 @@ def test_database_names_to_dump_bails_for_dry_run(): ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_and_capture_output').never() - names = module.database_names_to_dump({'name': 'all'}, {}, environment, dry_run=True) + names = module.database_names_to_dump( + {'name': 'all'}, {}, 'root', 'trustsome1', environment, dry_run=True + ) assert names == () @@ -31,12 +35,24 @@ def test_database_names_to_dump_queries_mysql_for_database_names(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_pipe' + ).with_args('root', 'trustsome1').and_return(99) flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('mysql', '--skip-column-names', '--batch', '--execute', 'show schemas'), + ( + 'mysql', + '--defaults-extra-file=/dev/fd/99', + '--skip-column-names', + '--batch', + '--execute', + 'show schemas', + ), environment=environment, ).and_return('foo\nbar\nmysql\n').once() - names = module.database_names_to_dump({'name': 'all'}, {}, environment, dry_run=False) + names = module.database_names_to_dump( + {'name': 'all'}, {}, 'root', 'trustsome1', environment, dry_run=False + ) assert names == ('foo', 'bar') @@ -56,6 +72,9 @@ def test_dump_data_sources_dumps_each_database(): databases = [{'name': 'foo'}, {'name': 'bar'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).and_return(None) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( ('bar',) @@ -65,6 +84,8 @@ def test_dump_data_sources_dumps_each_database(): flexmock(module).should_receive('execute_dump_command').with_args( database={'name': name}, config={}, + username=None, + password=None, dump_path=object, database_names=(name,), environment={'USER': 'root'}, @@ -89,10 +110,10 @@ def test_dump_data_sources_dumps_with_password(): database = {'name': 'foo', 'username': 'root', 'password': 'trustsome1'} process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') - flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( ('bar',) ) @@ -100,9 +121,11 @@ def test_dump_data_sources_dumps_with_password(): flexmock(module).should_receive('execute_dump_command').with_args( database=database, config={}, + username='root', + password='trustsome1', dump_path=object, database_names=('foo',), - environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'}, + environment={'USER': 'root'}, dry_run=object, dry_run_label=object, ).and_return(process).once() @@ -121,11 +144,16 @@ def test_dump_data_sources_dumps_all_databases_at_once(): databases = [{'name': 'all'}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).and_return(None) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) flexmock(module).should_receive('execute_dump_command').with_args( database={'name': 'all'}, config={}, + username=None, + password=None, dump_path=object, database_names=('foo', 'bar'), environment={'USER': 'root'}, @@ -147,6 +175,9 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured databases = [{'name': 'all', 'format': 'sql'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).and_return(None) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) @@ -154,6 +185,8 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured flexmock(module).should_receive('execute_dump_command').with_args( database={'name': name, 'format': 'sql'}, config={}, + username=None, + password=None, dump_path=object, database_names=(name,), environment={'USER': 'root'}, @@ -175,11 +208,15 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured def test_database_names_to_dump_runs_mysql_with_list_options(): - database = {'name': 'all', 'list_options': '--defaults-extra-file=my.cnf'} + database = {'name': 'all', 'list_options': '--defaults-file=my.cnf'} + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_pipe' + ).with_args('root', 'trustsome1').and_return(99) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ( 'mysql', - '--defaults-extra-file=my.cnf', + '--defaults-extra-file=/dev/fd/99', + '--defaults-file=my.cnf', '--skip-column-names', '--batch', '--execute', @@ -188,20 +225,27 @@ def test_database_names_to_dump_runs_mysql_with_list_options(): environment=None, ).and_return(('foo\nbar')).once() - assert module.database_names_to_dump(database, {}, None, '') == ('foo', 'bar') + assert module.database_names_to_dump(database, {}, 'root', 'trustsome1', None, '') == ( + 'foo', + 'bar', + ) def test_database_names_to_dump_runs_non_default_mysql_with_list_options(): database = { 'name': 'all', - 'list_options': '--defaults-extra-file=my.cnf', + 'list_options': '--defaults-file=my.cnf', 'mysql_command': 'custom_mysql', } + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_pipe' + ).with_args('root', 'trustsome1').and_return(99) flexmock(module).should_receive('execute_command_and_capture_output').with_args( environment=None, full_command=( 'custom_mysql', # Custom MySQL command - '--defaults-extra-file=my.cnf', + '--defaults-extra-file=/dev/fd/99', + '--defaults-file=my.cnf', '--skip-column-names', '--batch', '--execute', @@ -209,7 +253,10 @@ def test_database_names_to_dump_runs_non_default_mysql_with_list_options(): ), ).and_return(('foo\nbar')).once() - assert module.database_names_to_dump(database, {}, None, '') == ('foo', 'bar') + assert module.database_names_to_dump(database, {}, 'root', 'trustsome1', None, '') == ( + 'foo', + 'bar', + ) def test_execute_dump_command_runs_mysqldump(): @@ -219,11 +266,15 @@ def test_execute_dump_command_runs_mysqldump(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_pipe' + ).with_args('root', 'trustsome1').and_return(99) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mysqldump', + '--defaults-extra-file=/dev/fd/99', '--add-drop-database', '--databases', 'foo', @@ -238,6 +289,8 @@ def test_execute_dump_command_runs_mysqldump(): module.execute_dump_command( database={'name': 'foo'}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -255,11 +308,15 @@ def test_execute_dump_command_runs_mysqldump_without_add_drop_database(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_pipe' + ).with_args('root', 'trustsome1').and_return(99) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mysqldump', + '--defaults-extra-file=/dev/fd/99', '--databases', 'foo', '--result-file', @@ -273,6 +330,8 @@ def test_execute_dump_command_runs_mysqldump_without_add_drop_database(): module.execute_dump_command( database={'name': 'foo', 'add_drop_database': False}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -290,11 +349,15 @@ def test_execute_dump_command_runs_mysqldump_with_hostname_and_port(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_pipe' + ).with_args('root', 'trustsome1').and_return(99) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mysqldump', + '--defaults-extra-file=/dev/fd/99', '--add-drop-database', '--host', 'database.example.org', @@ -315,6 +378,8 @@ def test_execute_dump_command_runs_mysqldump_with_hostname_and_port(): module.execute_dump_command( database={'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -332,20 +397,22 @@ def test_execute_dump_command_runs_mysqldump_with_username_and_password(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_pipe' + ).with_args('root', 'trustsome1').and_return(99) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mysqldump', + '--defaults-extra-file=/dev/fd/99', '--add-drop-database', - '--user', - 'root', '--databases', 'foo', '--result-file', 'dump', ), - environment={'MYSQL_PWD': 'trustsome1'}, + environment={}, run_to_completion=False, ).and_return(process).once() @@ -353,9 +420,11 @@ def test_execute_dump_command_runs_mysqldump_with_username_and_password(): module.execute_dump_command( database={'name': 'foo', 'username': 'root', 'password': 'trustsome1'}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), - environment={'MYSQL_PWD': 'trustsome1'}, + environment={}, dry_run=False, dry_run_label='', ) @@ -370,11 +439,15 @@ def test_execute_dump_command_runs_mysqldump_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_pipe' + ).with_args('root', 'trustsome1').and_return(99) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mysqldump', + '--defaults-extra-file=/dev/fd/99', '--stuff=such', '--add-drop-database', '--databases', @@ -390,6 +463,8 @@ def test_execute_dump_command_runs_mysqldump_with_options(): module.execute_dump_command( database={'name': 'foo', 'options': '--stuff=such'}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -407,11 +482,15 @@ def test_execute_dump_command_runs_non_default_mysqldump(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_pipe' + ).with_args('root', 'trustsome1').and_return(99) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'custom_mysqldump', # Custom MySQL dump command + '--defaults-extra-file=/dev/fd/99', '--add-drop-database', '--databases', 'foo', @@ -429,6 +508,8 @@ def test_execute_dump_command_runs_non_default_mysqldump(): 'mysql_dump_command': 'custom_mysqldump', }, # Custom MySQL dump command specified config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -442,6 +523,9 @@ def test_execute_dump_command_runs_non_default_mysqldump(): def test_execute_dump_command_with_duplicate_dump_skips_mysqldump(): flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_pipe' + ).with_args('root', 'trustsome1').and_return(99) flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() @@ -449,6 +533,8 @@ def test_execute_dump_command_with_duplicate_dump_skips_mysqldump(): module.execute_dump_command( database={'name': 'foo'}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -465,6 +551,9 @@ def test_execute_dump_command_with_dry_run_skips_mysqldump(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_pipe' + ).with_args('root', 'trustsome1').and_return(99) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').never() @@ -473,6 +562,8 @@ def test_execute_dump_command_with_dry_run_skips_mysqldump(): module.execute_dump_command( database={'name': 'foo'}, config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), environment=None, @@ -675,13 +766,16 @@ def test_restore_data_source_dump_runs_mysql_with_username_and_password(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_pipe' + ).with_args('root', 'trustsome1').and_return(99) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( - ('mysql', '--batch', '--user', 'root'), + ('mysql', '--defaults-extra-file=/dev/fd/99', '--batch'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'}, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -717,10 +811,14 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_pipe' + ).with_args('cliusername', 'clipassword').and_return(99) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mysql', + '--defaults-extra-file=/dev/fd/99', '--batch', '--host', 'clihost', @@ -728,13 +826,11 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ 'cliport', '--protocol', 'tcp', - '--user', - 'cliusername', ), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - environment={'USER': 'root', 'MYSQL_PWD': 'clipassword'}, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -772,10 +868,14 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_pipe' + ).with_args('restoreuser', 'restorepass').and_return(99) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mysql', + '--defaults-extra-file=/dev/fd/99', '--batch', '--host', 'restorehost', @@ -783,13 +883,11 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ 'restoreport', '--protocol', 'tcp', - '--user', - 'restoreuser', ), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - environment={'USER': 'root', 'MYSQL_PWD': 'restorepass'}, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( diff --git a/tests/unit/test_execute.py b/tests/unit/test_execute.py index 4d39fe3b..f3ef9413 100644 --- a/tests/unit/test_execute.py +++ b/tests/unit/test_execute.py @@ -191,7 +191,7 @@ def test_execute_command_calls_full_command(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -213,7 +213,7 @@ def test_execute_command_calls_full_command_with_output_file(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stderr=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -234,7 +234,7 @@ def test_execute_command_calls_full_command_without_capturing_output(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(wait=lambda: 0)).once() flexmock(module).should_receive('interpret_exit_code').and_return(module.Exit_status.SUCCESS) flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) @@ -257,7 +257,7 @@ def test_execute_command_calls_full_command_with_input_file(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -278,7 +278,7 @@ def test_execute_command_calls_full_command_with_shell(): shell=True, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -299,7 +299,7 @@ def test_execute_command_calls_full_command_with_environment(): shell=False, env={'a': 'b'}, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -320,33 +320,12 @@ def test_execute_command_calls_full_command_with_working_directory(): shell=False, env=None, cwd='/working', - close_fds=True, - ).and_return(flexmock(stdout=None)).once() - flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) - flexmock(module).should_receive('log_outputs') - - output = module.execute_command(full_command, working_directory='/working') - - assert output is None - - -def test_execute_command_with_BORG_PASSPHRASE_FD_leaves_file_descriptors_open(): - full_command = ['foo', 'bar'] - flexmock(module).should_receive('log_command') - flexmock(module.subprocess).should_receive('Popen').with_args( - full_command, - stdin=None, - stdout=module.subprocess.PIPE, - stderr=module.subprocess.STDOUT, - shell=False, - env={'BORG_PASSPHRASE_FD': '4'}, - cwd=None, close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') - output = module.execute_command(full_command, environment={'BORG_PASSPHRASE_FD': '4'}) + output = module.execute_command(full_command, working_directory='/working') assert output is None @@ -363,7 +342,7 @@ def test_execute_command_without_run_to_completion_returns_process(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(process).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -377,11 +356,12 @@ def test_execute_command_and_capture_output_returns_stdout(): flexmock(module).should_receive('log_command') flexmock(module.subprocess).should_receive('check_output').with_args( full_command, + stdin=None, stderr=None, shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(decode=lambda: expected_output)).once() output = module.execute_command_and_capture_output(full_command) @@ -395,11 +375,12 @@ def test_execute_command_and_capture_output_with_capture_stderr_returns_stderr() flexmock(module).should_receive('log_command') flexmock(module.subprocess).should_receive('check_output').with_args( full_command, + stdin=None, stderr=module.subprocess.STDOUT, shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(decode=lambda: expected_output)).once() output = module.execute_command_and_capture_output(full_command, capture_stderr=True) @@ -414,11 +395,12 @@ def test_execute_command_and_capture_output_returns_output_when_process_error_is flexmock(module).should_receive('log_command') flexmock(module.subprocess).should_receive('check_output').with_args( full_command, + stdin=None, stderr=None, shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_raise(subprocess.CalledProcessError(1, full_command, err_output)).once() flexmock(module).should_receive('interpret_exit_code').and_return( module.Exit_status.SUCCESS @@ -435,11 +417,12 @@ def test_execute_command_and_capture_output_raises_when_command_errors(): flexmock(module).should_receive('log_command') flexmock(module.subprocess).should_receive('check_output').with_args( full_command, + stdin=None, stderr=None, shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_raise(subprocess.CalledProcessError(2, full_command, expected_output)).once() flexmock(module).should_receive('interpret_exit_code').and_return( module.Exit_status.ERROR @@ -455,11 +438,12 @@ def test_execute_command_and_capture_output_returns_output_with_shell(): flexmock(module).should_receive('log_command') flexmock(module.subprocess).should_receive('check_output').with_args( 'foo bar', + stdin=None, stderr=None, shell=True, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(decode=lambda: expected_output)).once() output = module.execute_command_and_capture_output(full_command, shell=True) @@ -473,11 +457,12 @@ def test_execute_command_and_capture_output_returns_output_with_environment(): flexmock(module).should_receive('log_command') flexmock(module.subprocess).should_receive('check_output').with_args( full_command, + stdin=None, stderr=None, shell=False, env={'a': 'b'}, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(decode=lambda: expected_output)).once() output = module.execute_command_and_capture_output( @@ -493,37 +478,16 @@ def test_execute_command_and_capture_output_returns_output_with_working_director flexmock(module).should_receive('log_command') flexmock(module.subprocess).should_receive('check_output').with_args( full_command, + stdin=None, stderr=None, shell=False, env=None, cwd='/working', - close_fds=True, - ).and_return(flexmock(decode=lambda: expected_output)).once() - - output = module.execute_command_and_capture_output( - full_command, shell=False, working_directory='/working' - ) - - assert output == expected_output - - -def test_execute_command_and_capture_output_with_BORG_PASSPHRASE_FD_leaves_file_descriptors_open(): - full_command = ['foo', 'bar'] - expected_output = '[]' - flexmock(module).should_receive('log_command') - flexmock(module.subprocess).should_receive('check_output').with_args( - full_command, - stderr=None, - shell=False, - env={'BORG_PASSPHRASE_FD': '4'}, - cwd=None, close_fds=False, ).and_return(flexmock(decode=lambda: expected_output)).once() output = module.execute_command_and_capture_output( - full_command, - shell=False, - environment={'BORG_PASSPHRASE_FD': '4'}, + full_command, shell=False, working_directory='/working' ) assert output == expected_output @@ -541,7 +505,7 @@ def test_execute_command_with_processes_calls_full_command(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -564,7 +528,7 @@ def test_execute_command_with_processes_returns_output_with_output_log_level_non shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(process).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs').and_return({process: 'out'}) @@ -587,7 +551,7 @@ def test_execute_command_with_processes_calls_full_command_with_output_file(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stderr=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -609,7 +573,7 @@ def test_execute_command_with_processes_calls_full_command_without_capturing_out shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(wait=lambda: 0)).once() flexmock(module).should_receive('interpret_exit_code').and_return(module.Exit_status.SUCCESS) flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) @@ -635,7 +599,7 @@ def test_execute_command_with_processes_calls_full_command_with_input_file(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -657,7 +621,7 @@ def test_execute_command_with_processes_calls_full_command_with_shell(): shell=True, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -679,7 +643,7 @@ def test_execute_command_with_processes_calls_full_command_with_environment(): shell=False, env={'a': 'b'}, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -701,39 +665,13 @@ def test_execute_command_with_processes_calls_full_command_with_working_director shell=False, env=None, cwd='/working', - close_fds=True, - ).and_return(flexmock(stdout=None)).once() - flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) - flexmock(module).should_receive('log_outputs') - - output = module.execute_command_with_processes( - full_command, processes, working_directory='/working' - ) - - assert output is None - - -def test_execute_command_with_processes_with_BORG_PASSPHRASE_FD_leaves_file_descriptors_open(): - full_command = ['foo', 'bar'] - processes = (flexmock(),) - flexmock(module).should_receive('log_command') - flexmock(module.subprocess).should_receive('Popen').with_args( - full_command, - stdin=None, - stdout=module.subprocess.PIPE, - stderr=module.subprocess.STDOUT, - shell=False, - env={'BORG_PASSPHRASE_FD': '4'}, - cwd=None, close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') output = module.execute_command_with_processes( - full_command, - processes, - environment={'BORG_PASSPHRASE_FD': '4'}, + full_command, processes, working_directory='/working' ) assert output is None @@ -754,7 +692,7 @@ def test_execute_command_with_processes_kills_processes_on_error(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_raise(subprocess.CalledProcessError(1, full_command, 'error')).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs').never() From 1e274d7153b6ddce4710c3425d0a94f6587e4736 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 28 Feb 2025 08:59:38 -0800 Subject: [PATCH 060/226] Add some missing test mocking (#1009). --- tests/unit/hooks/data_source/test_mariadb.py | 15 +++++++++++++++ tests/unit/hooks/data_source/test_mysql.py | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py index 6db363b8..c9c675b1 100644 --- a/tests/unit/hooks/data_source/test_mariadb.py +++ b/tests/unit/hooks/data_source/test_mariadb.py @@ -631,6 +631,9 @@ def test_restore_data_source_dump_runs_mariadb_to_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive( + 'make_defaults_file_pipe' + ).with_args(None, None).and_return(None) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mariadb', '--batch'), @@ -663,6 +666,9 @@ def test_restore_data_source_dump_runs_mariadb_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive( + 'make_defaults_file_pipe' + ).with_args(None, None).and_return(None) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mariadb', '--batch', '--harder'), @@ -697,6 +703,9 @@ def test_restore_data_source_dump_runs_non_default_mariadb_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive( + 'make_defaults_file_pipe' + ).with_args(None, None).and_return(None) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('custom_mariadb', '--batch', '--harder'), @@ -729,6 +738,9 @@ def test_restore_data_source_dump_runs_mariadb_with_hostname_and_port(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive( + 'make_defaults_file_pipe' + ).with_args(None, None).and_return(None) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( @@ -916,6 +928,9 @@ def test_restore_data_source_dump_with_dry_run_skips_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive( + 'make_defaults_file_pipe' + ).with_args(None, None).and_return(None) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').never() diff --git a/tests/unit/hooks/data_source/test_mysql.py b/tests/unit/hooks/data_source/test_mysql.py index 0fb3f432..c7098e4a 100644 --- a/tests/unit/hooks/data_source/test_mysql.py +++ b/tests/unit/hooks/data_source/test_mysql.py @@ -629,6 +629,9 @@ def test_restore_data_source_dump_runs_mysql_to_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_pipe' + ).with_args(None, None).and_return(None) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mysql', '--batch'), @@ -661,6 +664,9 @@ def test_restore_data_source_dump_runs_mysql_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_pipe' + ).with_args(None, None).and_return(None) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mysql', '--batch', '--harder'), @@ -693,6 +699,9 @@ def test_restore_data_source_dump_runs_non_default_mysql_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_pipe' + ).with_args(None, None).and_return(None) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('custom_mysql', '--batch', '--harder'), @@ -725,6 +734,9 @@ def test_restore_data_source_dump_runs_mysql_with_hostname_and_port(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_pipe' + ).with_args(None, None).and_return(None) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( @@ -912,6 +924,9 @@ def test_restore_data_source_dump_with_dry_run_skips_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_pipe' + ).with_args(None, None).and_return(None) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').never() From 48a4fbaa89b443ac0871a639c674e50b23e5a7ae Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 28 Feb 2025 09:21:01 -0800 Subject: [PATCH 061/226] Add missing test coverage for defaults file function (#1009). --- tests/unit/hooks/data_source/test_mariadb.py | 78 ++++++++++++++++---- 1 file changed, 63 insertions(+), 15 deletions(-) diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py index c9c675b1..dc2c1a28 100644 --- a/tests/unit/hooks/data_source/test_mariadb.py +++ b/tests/unit/hooks/data_source/test_mariadb.py @@ -6,6 +6,54 @@ from flexmock import flexmock from borgmatic.hooks.data_source import mariadb as module +def test_make_defaults_file_pipe_without_username_or_password_bails(): + flexmock(module.os).should_receive('pipe').never() + + assert module.make_defaults_file_pipe(username=None, password=None) is None + + +def test_make_defaults_file_pipe_with_username_and_password_writes_them_to_file_descriptor(): + read_descriptor = flexmock() + write_descriptor = flexmock() + + flexmock(module.os).should_receive('pipe').and_return(read_descriptor, write_descriptor) + flexmock(module.os).should_receive('write').with_args( + write_descriptor, b'[client]\nuser=root\npassword=trustsome1' + ).once() + flexmock(module.os).should_receive('close') + flexmock(module.os).should_receive('set_inheritable') + + assert module.make_defaults_file_pipe(username='root', password='trustsome1') == read_descriptor + + +def test_make_defaults_file_pipe_with_username_only_writes_it_to_file_descriptor(): + read_descriptor = flexmock() + write_descriptor = flexmock() + + flexmock(module.os).should_receive('pipe').and_return(read_descriptor, write_descriptor) + flexmock(module.os).should_receive('write').with_args( + write_descriptor, b'[client]\nuser=root' + ).once() + flexmock(module.os).should_receive('close') + flexmock(module.os).should_receive('set_inheritable') + + assert module.make_defaults_file_pipe(username='root', password=None) == read_descriptor + + +def test_make_defaults_file_pipe_with_password_only_writes_it_to_file_descriptor(): + read_descriptor = flexmock() + write_descriptor = flexmock() + + flexmock(module.os).should_receive('pipe').and_return(read_descriptor, write_descriptor) + flexmock(module.os).should_receive('write').with_args( + write_descriptor, b'[client]\npassword=trustsome1' + ).once() + flexmock(module.os).should_receive('close') + flexmock(module.os).should_receive('set_inheritable') + + assert module.make_defaults_file_pipe(username=None, password='trustsome1') == read_descriptor + + def test_database_names_to_dump_passes_through_name(): environment = flexmock() @@ -631,9 +679,9 @@ def test_restore_data_source_dump_runs_mariadb_to_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive( - 'make_defaults_file_pipe' - ).with_args(None, None).and_return(None) + flexmock(module).should_receive('make_defaults_file_pipe').with_args(None, None).and_return( + None + ) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mariadb', '--batch'), @@ -666,9 +714,9 @@ def test_restore_data_source_dump_runs_mariadb_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive( - 'make_defaults_file_pipe' - ).with_args(None, None).and_return(None) + flexmock(module).should_receive('make_defaults_file_pipe').with_args(None, None).and_return( + None + ) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mariadb', '--batch', '--harder'), @@ -703,9 +751,9 @@ def test_restore_data_source_dump_runs_non_default_mariadb_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive( - 'make_defaults_file_pipe' - ).with_args(None, None).and_return(None) + flexmock(module).should_receive('make_defaults_file_pipe').with_args(None, None).and_return( + None + ) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('custom_mariadb', '--batch', '--harder'), @@ -738,9 +786,9 @@ def test_restore_data_source_dump_runs_mariadb_with_hostname_and_port(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive( - 'make_defaults_file_pipe' - ).with_args(None, None).and_return(None) + flexmock(module).should_receive('make_defaults_file_pipe').with_args(None, None).and_return( + None + ) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( @@ -928,9 +976,9 @@ def test_restore_data_source_dump_with_dry_run_skips_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive( - 'make_defaults_file_pipe' - ).with_args(None, None).and_return(None) + flexmock(module).should_receive('make_defaults_file_pipe').with_args(None, None).and_return( + None + ) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').never() From baf5fec78d5185a43738eda498575ca2d2347d36 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 28 Feb 2025 10:53:17 -0800 Subject: [PATCH 062/226] If the user supplies their own --defaults-extra-file, include it from the one we generate (#1009). --- borgmatic/hooks/data_source/mariadb.py | 152 +++++++++++++++++++------ borgmatic/hooks/data_source/mysql.py | 40 +++---- 2 files changed, 131 insertions(+), 61 deletions(-) diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index c57abf7a..4653319a 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -1,6 +1,7 @@ import copy import logging import os +import re import shlex import borgmatic.borg.pattern @@ -23,19 +24,46 @@ def make_dump_path(base_directory): # pragma: no cover return dump.make_data_source_dump_path(base_directory, 'mariadb_databases') -SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys') +DEFAULTS_EXTRA_FILE_FLAG_PATTERN = re.compile('^--defaults-extra-file=(?P.*)$') -def make_defaults_file_pipe(username=None, password=None): +def parse_extra_options(extra_options): ''' - Given a database username and/or password, write it to an anonymous pipe and return its file - descriptor for passing to an executed command. The idea is that this is a more secure way to - transmit credentials to a database client than using an environment variable. + Given an extra options string, split the options into a tuple and return it. Additionally, if + the first option is "--defaults-extra-file=...", then remove it from the options and return the + filename. - If no username or password are given, then return None. + So the return value is a tuple of: (parsed options, defaults extra filename). - Do not use this value for multiple different command invocations. That will not work because - each pipe is "used up" once read. + The intent is to support downstream merging of multiple "--defaults-extra-file"s, as + MariaDB/MySQL only allows one at a time. + ''' + split_extra_options = tuple(shlex.split(extra_options)) if extra_options else () + + if not split_extra_options: + return (split_extra_options, None) + + match = DEFAULTS_EXTRA_FILE_FLAG_PATTERN.match(split_extra_options[0]) + + if not match: + return (split_extra_options, None) + + return (split_extra_options[1:], match.group('filename')) + + +def make_defaults_file_options(username=None, password=None, defaults_extra_filename=None): + ''' + Given a database username and/or password, write it to an anonymous pipe and return the flags + for passing that file descriptor to an executed command. The idea is that this is a more secure + way to transmit credentials to a database client than using an environment variable. + + If no username or password are given, then return the options for the given defaults extra + filename (if any). But if there is a username and/or password and a defaults extra filename is + given, then "!include" it from the generated file, effectively allowing multiple defaults extra + files. + + Do not use the returned value for multiple different command invocations. That will not work + because each pipe is "used up" once read. ''' values = '\n'.join( ( @@ -45,7 +73,10 @@ def make_defaults_file_pipe(username=None, password=None): ).strip() if not values: - return None + if defaults_extra_filename: + return (f'--defaults-extra-file={defaults_extra_filename}',) + + return () fields_message = ' and '.join( field_name @@ -55,17 +86,20 @@ def make_defaults_file_pipe(username=None, password=None): ) if field_name is not None ) - logger.debug(f'Writing database {fields_message} to defaults extra file pipe') + include_message = f' (including {defaults_extra_filename})' if defaults_extra_filename else '' + logger.debug(f'Writing database {fields_message} to defaults extra file pipe{include_message}') + + include = f'!include {defaults_extra_filename}\n' if defaults_extra_filename else '' read_file_descriptor, write_file_descriptor = os.pipe() - os.write(write_file_descriptor, f'[client]\n{values}'.encode('utf-8')) + os.write(write_file_descriptor, f'{include}[client]\n{values}'.encode('utf-8')) os.close(write_file_descriptor) # This plus subprocess.Popen(..., close_fds=False) in execute.py is necessary for the database # client child process to inherit the file descriptor. os.set_inheritable(read_file_descriptor, True) - return read_file_descriptor + return (f'--defaults-extra-file=/dev/fd/{read_file_descriptor}',) def database_names_to_dump(database, config, username, password, environment, dry_run): @@ -83,15 +117,73 @@ def database_names_to_dump(database, config, username, password, environment, dr mariadb_show_command = tuple( shlex.quote(part) for part in shlex.split(database.get('mariadb_command') or 'mariadb') ) - defaults_file_descriptor = make_defaults_file_pipe(username, password) + extra_options, defaults_extra_filename = parse_extra_options(database.get('list_options')) show_command = ( mariadb_show_command - + ( - (f'--defaults-extra-file=/dev/fd/{defaults_file_descriptor}',) - if defaults_file_descriptor - else () + + make_defaults_file_options(username, password, defaults_extra_filename) + + extra_options + + (('--host', database['hostname']) if 'hostname' in database else ()) + + (('--port', str(database['port'])) if 'port' in database else ()) + + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + + ('--skip-column-names', '--batch') + + ('--execute', 'show schemas') + ) + + logger.debug('Querying for "all" MariaDB databases to dump') + + show_output = execute_command_and_capture_output(show_command, environment=environment) + + return tuple( + show_name + for show_name in show_output.strip().splitlines() + if show_name not in SYSTEM_DATABASE_NAMES + ) + + +SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys') + + +def execute_dump_command( + database, + config, + username, + password, + dump_path, + database_names, + environment, + dry_run, + dry_run_label, +): + ''' + Kick off a dump for the given MariaDB database (provided as a configuration dict) to a named + pipe constructed from the given dump path and database name. + + Return a subprocess.Popen instance for the dump process ready to spew to a named pipe. But if + this is a dry run, then don't actually dump anything and return None. + ''' + database_name = database['name'] + dump_filename = dump.make_data_source_dump_filename( + dump_path, + database['name'], + database.get('hostname'), + database.get('port'), + ) + + if os.path.exists(dump_filename): + logger.warning( + f'Skipping duplicate dump of MariaDB database "{database_name}" to {dump_filename}' ) - + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ()) + return None + + mariadb_dump_command = tuple( + shlex.quote(part) + for part in shlex.split(database.get('mariadb_dump_command') or 'mariadb-dump') + ) + extra_options, defaults_extra_filename = parse_extra_options(database.get('options')) + dump_command = ( + mariadb_dump_command + + make_defaults_file_options(username, password, defaults_extra_filename) + + extra_options + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) @@ -146,15 +238,11 @@ def execute_dump_command( shlex.quote(part) for part in shlex.split(database.get('mariadb_dump_command') or 'mariadb-dump') ) - defaults_file_descriptor = make_defaults_file_pipe(username, password) + extra_options, defaults_extra_filename = parse_extra_options(database.get('options')) dump_command = ( mariadb_dump_command - + ( - (f'--defaults-extra-file=/dev/fd/{defaults_file_descriptor}',) - if defaults_file_descriptor - else () - ) - + (tuple(database['options'].split(' ')) if 'options' in database else ()) + + make_defaults_file_options(username, password, defaults_extra_filename) + + extra_options + (('--add-drop-database',) if database.get('add_drop_database', True) else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) @@ -346,20 +434,12 @@ def restore_data_source_dump( mariadb_restore_command = tuple( shlex.quote(part) for part in shlex.split(data_source.get('mariadb_command') or 'mariadb') ) - defaults_file_descriptor = make_defaults_file_pipe(username, password) + extra_options, defaults_extra_filename = parse_extra_options(database.get('restore_options')) restore_command = ( mariadb_restore_command - + ( - (f'--defaults-extra-file=/dev/fd/{defaults_file_descriptor}',) - if defaults_file_descriptor - else () - ) + + make_defaults_file_options(username, password, defaults_extra_filename) + + extra_options + ('--batch',) - + ( - tuple(data_source['restore_options'].split(' ')) - if 'restore_options' in data_source - else () - ) + (('--host', hostname) if hostname else ()) + (('--port', str(port)) if port else ()) + (('--protocol', 'tcp') if hostname or port else ()) diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index a31f7a54..9c4685b7 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -42,17 +42,15 @@ def database_names_to_dump(database, config, username, password, environment, dr mysql_show_command = tuple( shlex.quote(part) for part in shlex.split(database.get('mysql_command') or 'mysql') ) - defaults_file_descriptor = borgmatic.hooks.data_source.mariadb.make_defaults_file_pipe( - username, password + extra_options, defaults_extra_filename = ( + borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('list_options')) ) show_command = ( mysql_show_command - + ( - (f'--defaults-extra-file=/dev/fd/{defaults_file_descriptor}',) - if defaults_file_descriptor - else () + + borgmatic.hooks.data_source.mariadb.make_defaults_file_options( + username, password, defaults_extra_filename ) - + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ()) + + extra_options + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) @@ -106,17 +104,15 @@ def execute_dump_command( mysql_dump_command = tuple( shlex.quote(part) for part in shlex.split(database.get('mysql_dump_command') or 'mysqldump') ) - defaults_file_descriptor = borgmatic.hooks.data_source.mariadb.make_defaults_file_pipe( - username, password + extra_options, defaults_extra_filename = ( + borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('options')) ) dump_command = ( mysql_dump_command - + ( - (f'--defaults-extra-file=/dev/fd/{defaults_file_descriptor}',) - if defaults_file_descriptor - else () + + borgmatic.hooks.data_source.mariadb.make_defaults_file_options( + username, password, defaults_extra_filename ) - + (tuple(database['options'].split(' ')) if 'options' in database else ()) + + extra_options + (('--add-drop-database',) if database.get('add_drop_database', True) else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) @@ -308,22 +304,16 @@ def restore_data_source_dump( mysql_restore_command = tuple( shlex.quote(part) for part in shlex.split(data_source.get('mysql_command') or 'mysql') ) - defaults_file_descriptor = borgmatic.hooks.data_source.mariadb.make_defaults_file_pipe( - username, password + extra_options, defaults_extra_filename = ( + borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('restore_options')) ) restore_command = ( mysql_restore_command - + ( - (f'--defaults-extra-file=/dev/fd/{defaults_file_descriptor}',) - if defaults_file_descriptor - else () + + borgmatic.hooks.data_source.mariadb.make_defaults_file_options( + username, password, defaults_extra_filename ) + + extra_options + ('--batch',) - + ( - tuple(data_source['restore_options'].split(' ')) - if 'restore_options' in data_source - else () - ) + (('--host', hostname) if hostname else ()) + (('--port', str(port)) if port else ()) + (('--protocol', 'tcp') if hostname or port else ()) From 1e5c256d54fa3edf88b80520c8df7049563a5f09 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 28 Feb 2025 14:40:00 -0800 Subject: [PATCH 063/226] Get tests passing again (#1009). --- borgmatic/hooks/data_source/mariadb.py | 4 +- borgmatic/hooks/data_source/mysql.py | 2 +- tests/unit/hooks/data_source/test_mariadb.py | 217 +++++++++++++------ tests/unit/hooks/data_source/test_mysql.py | 149 +++++++++---- 4 files changed, 254 insertions(+), 118 deletions(-) diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index 4653319a..985cd5b2 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -41,7 +41,7 @@ def parse_extra_options(extra_options): split_extra_options = tuple(shlex.split(extra_options)) if extra_options else () if not split_extra_options: - return (split_extra_options, None) + return ((), None) match = DEFAULTS_EXTRA_FILE_FLAG_PATTERN.match(split_extra_options[0]) @@ -434,7 +434,7 @@ def restore_data_source_dump( mariadb_restore_command = tuple( shlex.quote(part) for part in shlex.split(data_source.get('mariadb_command') or 'mariadb') ) - extra_options, defaults_extra_filename = parse_extra_options(database.get('restore_options')) + extra_options, defaults_extra_filename = parse_extra_options(data_source.get('restore_options')) restore_command = ( mariadb_restore_command + make_defaults_file_options(username, password, defaults_extra_filename) diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index 9c4685b7..13d77041 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -305,7 +305,7 @@ def restore_data_source_dump( shlex.quote(part) for part in shlex.split(data_source.get('mysql_command') or 'mysql') ) extra_options, defaults_extra_filename = ( - borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('restore_options')) + borgmatic.hooks.data_source.mariadb.parse_extra_options(data_source.get('restore_options')) ) restore_command = ( mysql_restore_command diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py index dc2c1a28..d425b82c 100644 --- a/tests/unit/hooks/data_source/test_mariadb.py +++ b/tests/unit/hooks/data_source/test_mariadb.py @@ -9,11 +9,11 @@ from borgmatic.hooks.data_source import mariadb as module def test_make_defaults_file_pipe_without_username_or_password_bails(): flexmock(module.os).should_receive('pipe').never() - assert module.make_defaults_file_pipe(username=None, password=None) is None + assert module.make_defaults_file_options(username=None, password=None) is () -def test_make_defaults_file_pipe_with_username_and_password_writes_them_to_file_descriptor(): - read_descriptor = flexmock() +def test_make_defaults_file_option_with_username_and_password_writes_them_to_file_descriptor(): + read_descriptor = 99 write_descriptor = flexmock() flexmock(module.os).should_receive('pipe').and_return(read_descriptor, write_descriptor) @@ -23,11 +23,11 @@ def test_make_defaults_file_pipe_with_username_and_password_writes_them_to_file_ flexmock(module.os).should_receive('close') flexmock(module.os).should_receive('set_inheritable') - assert module.make_defaults_file_pipe(username='root', password='trustsome1') == read_descriptor + assert module.make_defaults_file_options(username='root', password='trustsome1') == ('--defaults-extra-file=/dev/fd/99',) def test_make_defaults_file_pipe_with_username_only_writes_it_to_file_descriptor(): - read_descriptor = flexmock() + read_descriptor = 99 write_descriptor = flexmock() flexmock(module.os).should_receive('pipe').and_return(read_descriptor, write_descriptor) @@ -37,11 +37,11 @@ def test_make_defaults_file_pipe_with_username_only_writes_it_to_file_descriptor flexmock(module.os).should_receive('close') flexmock(module.os).should_receive('set_inheritable') - assert module.make_defaults_file_pipe(username='root', password=None) == read_descriptor + assert module.make_defaults_file_options(username='root', password=None) == ('--defaults-extra-file=/dev/fd/99',) def test_make_defaults_file_pipe_with_password_only_writes_it_to_file_descriptor(): - read_descriptor = flexmock() + read_descriptor = 99 write_descriptor = flexmock() flexmock(module.os).should_receive('pipe').and_return(read_descriptor, write_descriptor) @@ -51,7 +51,21 @@ def test_make_defaults_file_pipe_with_password_only_writes_it_to_file_descriptor flexmock(module.os).should_receive('close') flexmock(module.os).should_receive('set_inheritable') - assert module.make_defaults_file_pipe(username=None, password='trustsome1') == read_descriptor + assert module.make_defaults_file_options(username=None, password='trustsome1') == ('--defaults-extra-file=/dev/fd/99',) + + +def test_make_defaults_file_option_with_defaults_extra_filename_includes_it_in_file_descriptor(): + read_descriptor = 99 + write_descriptor = flexmock() + + flexmock(module.os).should_receive('pipe').and_return(read_descriptor, write_descriptor) + flexmock(module.os).should_receive('write').with_args( + write_descriptor, b'!include extra.cnf\n[client]\nuser=root\npassword=trustsome1' + ).once() + flexmock(module.os).should_receive('close') + flexmock(module.os).should_receive('set_inheritable') + + assert module.make_defaults_file_options(username='root', password='trustsome1', defaults_extra_filename='extra.cnf') == ('--defaults-extra-file=/dev/fd/99',) def test_database_names_to_dump_passes_through_name(): @@ -80,9 +94,12 @@ def test_database_names_to_dump_queries_mariadb_for_database_names(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_defaults_file_pipe').with_args( - 'root', 'trustsome1' - ).and_return(99) + flexmock(module).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ( 'mariadb', @@ -256,15 +273,18 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured def test_database_names_to_dump_runs_mariadb_with_list_options(): - database = {'name': 'all', 'list_options': '--defaults-file=mariadb.cnf'} - flexmock(module).should_receive('make_defaults_file_pipe').with_args( - 'root', 'trustsome1' - ).and_return(99) + database = {'name': 'all', 'list_options': '--defaults-extra-file=mariadb.cnf --skip-ssl'} + flexmock(module).should_receive( + 'parse_extra_options' + ).and_return(('--skip-ssl',), 'mariadb.cnf') + flexmock(module).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', 'mariadb.cnf').and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ( 'mariadb', '--defaults-extra-file=/dev/fd/99', - '--defaults-file=mariadb.cnf', + '--skip-ssl', '--skip-column-names', '--batch', '--execute', @@ -282,18 +302,21 @@ def test_database_names_to_dump_runs_mariadb_with_list_options(): def test_database_names_to_dump_runs_non_default_mariadb_with_list_options(): database = { 'name': 'all', - 'list_options': '--defaults-file=mariadb.cnf', + 'list_options': '--defaults-extra-file=mariadb.cnf --skip-ssl', 'mariadb_command': 'custom_mariadb', } - flexmock(module).should_receive('make_defaults_file_pipe').with_args( - 'root', 'trustsome1' - ).and_return(99) + flexmock(module).should_receive( + 'parse_extra_options' + ).and_return(('--skip-ssl',), 'mariadb.cnf') + flexmock(module).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', 'mariadb.cnf').and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module).should_receive('execute_command_and_capture_output').with_args( environment=None, full_command=( 'custom_mariadb', # Custom MariaDB command '--defaults-extra-file=/dev/fd/99', - '--defaults-file=mariadb.cnf', + '--skip-ssl', '--skip-column-names', '--batch', '--execute', @@ -314,9 +337,12 @@ def test_execute_dump_command_runs_mariadb_dump(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_defaults_file_pipe').with_args( - 'root', 'trustsome1' - ).and_return(99) + flexmock(module).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -356,9 +382,12 @@ def test_execute_dump_command_runs_mariadb_dump_without_add_drop_database(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_defaults_file_pipe').with_args( - 'root', 'trustsome1' - ).and_return(99) + flexmock(module).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -397,9 +426,12 @@ def test_execute_dump_command_runs_mariadb_dump_with_hostname_and_port(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_defaults_file_pipe').with_args( - 'root', 'trustsome1' - ).and_return(99) + flexmock(module).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -445,9 +477,12 @@ def test_execute_dump_command_runs_mariadb_dump_with_username_and_password(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_defaults_file_pipe').with_args( - 'root', 'trustsome1' - ).and_return(99) + flexmock(module).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -487,9 +522,12 @@ def test_execute_dump_command_runs_mariadb_dump_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_defaults_file_pipe').with_args( - 'root', 'trustsome1' - ).and_return(99) + flexmock(module).should_receive( + 'parse_extra_options' + ).and_return(('--stuff=such',), None) + flexmock(module).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -530,9 +568,12 @@ def test_execute_dump_command_runs_non_default_mariadb_dump_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_defaults_file_pipe').with_args( - 'root', 'trustsome1' - ).and_return(99) + flexmock(module).should_receive( + 'parse_extra_options' + ).and_return(('--stuff=such',), None) + flexmock(module).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -573,9 +614,12 @@ def test_execute_dump_command_runs_non_default_mariadb_dump_with_options(): def test_execute_dump_command_with_duplicate_dump_skips_mariadb_dump(): flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(True) - flexmock(module).should_receive('make_defaults_file_pipe').with_args( - 'root', 'trustsome1' - ).and_return(99) + flexmock(module).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() @@ -601,9 +645,12 @@ def test_execute_dump_command_with_dry_run_skips_mariadb_dump(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_defaults_file_pipe').with_args( - 'root', 'trustsome1' - ).and_return(99) + flexmock(module).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').never() @@ -679,9 +726,12 @@ def test_restore_data_source_dump_runs_mariadb_to_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_defaults_file_pipe').with_args(None, None).and_return( - None - ) + flexmock(module).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mariadb', '--batch'), @@ -714,12 +764,15 @@ def test_restore_data_source_dump_runs_mariadb_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_defaults_file_pipe').with_args(None, None).and_return( - None - ) + flexmock(module).should_receive( + 'parse_extra_options' + ).and_return(('--harder',), None) + flexmock(module).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( - ('mariadb', '--batch', '--harder'), + ('mariadb', '--harder', '--batch'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, @@ -751,12 +804,15 @@ def test_restore_data_source_dump_runs_non_default_mariadb_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_defaults_file_pipe').with_args(None, None).and_return( - None - ) + flexmock(module).should_receive( + 'parse_extra_options' + ).and_return(('--harder',), None) + flexmock(module).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( - ('custom_mariadb', '--batch', '--harder'), + ('custom_mariadb', '--harder', '--batch'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, @@ -786,9 +842,12 @@ def test_restore_data_source_dump_runs_mariadb_with_hostname_and_port(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_defaults_file_pipe').with_args(None, None).and_return( - None - ) + flexmock(module).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( @@ -830,9 +889,12 @@ def test_restore_data_source_dump_runs_mariadb_with_username_and_password(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_defaults_file_pipe').with_args( - 'root', 'trustsome1' - ).and_return(99) + flexmock(module).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mariadb', '--defaults-extra-file=/dev/fd/99', '--batch'), @@ -875,9 +937,14 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_defaults_file_pipe').with_args( - 'cliusername', 'clipassword' - ).and_return(99) + flexmock(module).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module).should_receive( + 'make_defaults_file_options' + ).with_args('cliusername', 'clipassword', None).and_return( + ('--defaults-extra-file=/dev/fd/99',) + ) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( @@ -932,9 +999,14 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_defaults_file_pipe').with_args( - 'restoreuser', 'restorepass' - ).and_return(99) + flexmock(module).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module).should_receive( + 'make_defaults_file_options' + ).with_args('restoreuser', 'restorepass', None).and_return( + ('--defaults-extra-file=/dev/fd/99',) + ) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( @@ -976,9 +1048,12 @@ def test_restore_data_source_dump_with_dry_run_skips_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive('make_defaults_file_pipe').with_args(None, None).and_return( - None - ) + flexmock(module).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').never() diff --git a/tests/unit/hooks/data_source/test_mysql.py b/tests/unit/hooks/data_source/test_mysql.py index c7098e4a..2e629dc3 100644 --- a/tests/unit/hooks/data_source/test_mysql.py +++ b/tests/unit/hooks/data_source/test_mysql.py @@ -36,8 +36,11 @@ def test_database_names_to_dump_queries_mysql_for_database_names(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( - 'make_defaults_file_pipe' - ).with_args('root', 'trustsome1').and_return(99) + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ( 'mysql', @@ -208,15 +211,18 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured def test_database_names_to_dump_runs_mysql_with_list_options(): - database = {'name': 'all', 'list_options': '--defaults-file=my.cnf'} + database = {'name': 'all', 'list_options': '--defaults-extra-file=my.cnf --skip-ssl'} flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( - 'make_defaults_file_pipe' - ).with_args('root', 'trustsome1').and_return(99) + 'parse_extra_options' + ).and_return(('--skip-ssl',), 'my.cnf') + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', 'my.cnf').and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ( 'mysql', '--defaults-extra-file=/dev/fd/99', - '--defaults-file=my.cnf', + '--skip-ssl', '--skip-column-names', '--batch', '--execute', @@ -234,18 +240,21 @@ def test_database_names_to_dump_runs_mysql_with_list_options(): def test_database_names_to_dump_runs_non_default_mysql_with_list_options(): database = { 'name': 'all', - 'list_options': '--defaults-file=my.cnf', + 'list_options': '--defaults-extra-file=my.cnf --skip-ssl', 'mysql_command': 'custom_mysql', } flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( - 'make_defaults_file_pipe' - ).with_args('root', 'trustsome1').and_return(99) + 'parse_extra_options' + ).and_return(('--skip-ssl',), 'my.cnf') + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', 'my.cnf').and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module).should_receive('execute_command_and_capture_output').with_args( environment=None, full_command=( 'custom_mysql', # Custom MySQL command '--defaults-extra-file=/dev/fd/99', - '--defaults-file=my.cnf', + '--skip-ssl', '--skip-column-names', '--batch', '--execute', @@ -267,8 +276,11 @@ def test_execute_dump_command_runs_mysqldump(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( - 'make_defaults_file_pipe' - ).with_args('root', 'trustsome1').and_return(99) + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -309,8 +321,11 @@ def test_execute_dump_command_runs_mysqldump_without_add_drop_database(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( - 'make_defaults_file_pipe' - ).with_args('root', 'trustsome1').and_return(99) + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -350,8 +365,11 @@ def test_execute_dump_command_runs_mysqldump_with_hostname_and_port(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( - 'make_defaults_file_pipe' - ).with_args('root', 'trustsome1').and_return(99) + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -398,8 +416,11 @@ def test_execute_dump_command_runs_mysqldump_with_username_and_password(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( - 'make_defaults_file_pipe' - ).with_args('root', 'trustsome1').and_return(99) + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -440,8 +461,11 @@ def test_execute_dump_command_runs_mysqldump_with_options(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( - 'make_defaults_file_pipe' - ).with_args('root', 'trustsome1').and_return(99) + 'parse_extra_options' + ).and_return(('--stuff=such',), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -483,8 +507,11 @@ def test_execute_dump_command_runs_non_default_mysqldump(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( - 'make_defaults_file_pipe' - ).with_args('root', 'trustsome1').and_return(99) + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -524,8 +551,11 @@ def test_execute_dump_command_with_duplicate_dump_skips_mysqldump(): flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(True) flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( - 'make_defaults_file_pipe' - ).with_args('root', 'trustsome1').and_return(99) + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() @@ -552,8 +582,11 @@ def test_execute_dump_command_with_dry_run_skips_mysqldump(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( - 'make_defaults_file_pipe' - ).with_args('root', 'trustsome1').and_return(99) + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').never() @@ -630,8 +663,11 @@ def test_restore_data_source_dump_runs_mysql_to_restore(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( - 'make_defaults_file_pipe' - ).with_args(None, None).and_return(None) + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mysql', '--batch'), @@ -665,11 +701,14 @@ def test_restore_data_source_dump_runs_mysql_with_options(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( - 'make_defaults_file_pipe' - ).with_args(None, None).and_return(None) + 'parse_extra_options' + ).and_return(('--harder',), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( - ('mysql', '--batch', '--harder'), + ('mysql', '--harder', '--batch'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, @@ -700,11 +739,14 @@ def test_restore_data_source_dump_runs_non_default_mysql_with_options(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( - 'make_defaults_file_pipe' - ).with_args(None, None).and_return(None) + 'parse_extra_options' + ).and_return(('--harder',), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( - ('custom_mysql', '--batch', '--harder'), + ('custom_mysql', '--harder', '--batch'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, @@ -735,8 +777,11 @@ def test_restore_data_source_dump_runs_mysql_with_hostname_and_port(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( - 'make_defaults_file_pipe' - ).with_args(None, None).and_return(None) + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( @@ -779,8 +824,11 @@ def test_restore_data_source_dump_runs_mysql_with_username_and_password(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( - 'make_defaults_file_pipe' - ).with_args('root', 'trustsome1').and_return(99) + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mysql', '--defaults-extra-file=/dev/fd/99', '--batch'), @@ -824,8 +872,13 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( - 'make_defaults_file_pipe' - ).with_args('cliusername', 'clipassword').and_return(99) + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('cliusername', 'clipassword', None).and_return( + ('--defaults-extra-file=/dev/fd/99',) + ) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( @@ -881,8 +934,13 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( - 'make_defaults_file_pipe' - ).with_args('restoreuser', 'restorepass').and_return(99) + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('restoreuser', 'restorepass', None).and_return( + ('--defaults-extra-file=/dev/fd/99',) + ) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( @@ -925,8 +983,11 @@ def test_restore_data_source_dump_with_dry_run_skips_restore(): 'resolve_credential' ).replace_with(lambda value, config: value) flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( - 'make_defaults_file_pipe' - ).with_args(None, None).and_return(None) + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').never() From 06b065cb09fef01e14375bca8cfe34f4ecfae2b2 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 28 Feb 2025 18:28:09 -0800 Subject: [PATCH 064/226] Add missing test coverage (#1009). --- borgmatic/hooks/data_source/mariadb.py | 59 ----- tests/unit/hooks/data_source/test_mariadb.py | 246 +++++++++---------- 2 files changed, 121 insertions(+), 184 deletions(-) diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index 985cd5b2..fc879142 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -143,65 +143,6 @@ def database_names_to_dump(database, config, username, password, environment, dr SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys') -def execute_dump_command( - database, - config, - username, - password, - dump_path, - database_names, - environment, - dry_run, - dry_run_label, -): - ''' - Kick off a dump for the given MariaDB database (provided as a configuration dict) to a named - pipe constructed from the given dump path and database name. - - Return a subprocess.Popen instance for the dump process ready to spew to a named pipe. But if - this is a dry run, then don't actually dump anything and return None. - ''' - database_name = database['name'] - dump_filename = dump.make_data_source_dump_filename( - dump_path, - database['name'], - database.get('hostname'), - database.get('port'), - ) - - if os.path.exists(dump_filename): - logger.warning( - f'Skipping duplicate dump of MariaDB database "{database_name}" to {dump_filename}' - ) - return None - - mariadb_dump_command = tuple( - shlex.quote(part) - for part in shlex.split(database.get('mariadb_dump_command') or 'mariadb-dump') - ) - extra_options, defaults_extra_filename = parse_extra_options(database.get('options')) - dump_command = ( - mariadb_dump_command - + make_defaults_file_options(username, password, defaults_extra_filename) - + extra_options - + (('--host', database['hostname']) if 'hostname' in database else ()) - + (('--port', str(database['port'])) if 'port' in database else ()) - + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) - + ('--skip-column-names', '--batch') - + ('--execute', 'show schemas') - ) - - logger.debug('Querying for "all" MariaDB databases to dump') - - show_output = execute_command_and_capture_output(show_command, environment=environment) - - return tuple( - show_name - for show_name in show_output.strip().splitlines() - if show_name not in SYSTEM_DATABASE_NAMES - ) - - def execute_dump_command( database, config, diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py index d425b82c..fe183865 100644 --- a/tests/unit/hooks/data_source/test_mariadb.py +++ b/tests/unit/hooks/data_source/test_mariadb.py @@ -6,10 +6,28 @@ from flexmock import flexmock from borgmatic.hooks.data_source import mariadb as module +def test_parse_extra_options_passes_through_empty_options(): + assert module.parse_extra_options('') == ((), None) + + +def test_parse_extra_options_with_defaults_extra_file_removes_and_and_parses_out_filename(): + assert module.parse_extra_options('--defaults-extra-file=extra.cnf --skip-ssl') == ( + ('--skip-ssl',), + 'extra.cnf', + ) + + +def test_parse_extra_options_without_defaults_extra_file_passes_through_options(): + assert module.parse_extra_options('--skip-ssl --and=stuff') == ( + ('--skip-ssl', '--and=stuff'), + None, + ) + + def test_make_defaults_file_pipe_without_username_or_password_bails(): flexmock(module.os).should_receive('pipe').never() - assert module.make_defaults_file_options(username=None, password=None) is () + assert module.make_defaults_file_options(username=None, password=None) == () def test_make_defaults_file_option_with_username_and_password_writes_them_to_file_descriptor(): @@ -23,10 +41,12 @@ def test_make_defaults_file_option_with_username_and_password_writes_them_to_fil flexmock(module.os).should_receive('close') flexmock(module.os).should_receive('set_inheritable') - assert module.make_defaults_file_options(username='root', password='trustsome1') == ('--defaults-extra-file=/dev/fd/99',) + assert module.make_defaults_file_options(username='root', password='trustsome1') == ( + '--defaults-extra-file=/dev/fd/99', + ) -def test_make_defaults_file_pipe_with_username_only_writes_it_to_file_descriptor(): +def test_make_defaults_file_pipe_with_only_username_writes_it_to_file_descriptor(): read_descriptor = 99 write_descriptor = flexmock() @@ -37,10 +57,12 @@ def test_make_defaults_file_pipe_with_username_only_writes_it_to_file_descriptor flexmock(module.os).should_receive('close') flexmock(module.os).should_receive('set_inheritable') - assert module.make_defaults_file_options(username='root', password=None) == ('--defaults-extra-file=/dev/fd/99',) + assert module.make_defaults_file_options(username='root', password=None) == ( + '--defaults-extra-file=/dev/fd/99', + ) -def test_make_defaults_file_pipe_with_password_only_writes_it_to_file_descriptor(): +def test_make_defaults_file_pipe_with_only_password_writes_it_to_file_descriptor(): read_descriptor = 99 write_descriptor = flexmock() @@ -51,7 +73,9 @@ def test_make_defaults_file_pipe_with_password_only_writes_it_to_file_descriptor flexmock(module.os).should_receive('close') flexmock(module.os).should_receive('set_inheritable') - assert module.make_defaults_file_options(username=None, password='trustsome1') == ('--defaults-extra-file=/dev/fd/99',) + assert module.make_defaults_file_options(username=None, password='trustsome1') == ( + '--defaults-extra-file=/dev/fd/99', + ) def test_make_defaults_file_option_with_defaults_extra_filename_includes_it_in_file_descriptor(): @@ -65,7 +89,17 @@ def test_make_defaults_file_option_with_defaults_extra_filename_includes_it_in_f flexmock(module.os).should_receive('close') flexmock(module.os).should_receive('set_inheritable') - assert module.make_defaults_file_options(username='root', password='trustsome1', defaults_extra_filename='extra.cnf') == ('--defaults-extra-file=/dev/fd/99',) + assert module.make_defaults_file_options( + username='root', password='trustsome1', defaults_extra_filename='extra.cnf' + ) == ('--defaults-extra-file=/dev/fd/99',) + + +def test_make_defaults_file_option_with_only_defaults_extra_filename_uses_it_instead_of_file_descriptor(): + flexmock(module.os).should_receive('pipe').never() + + assert module.make_defaults_file_options( + username=None, password=None, defaults_extra_filename='extra.cnf' + ) == ('--defaults-extra-file=extra.cnf',) def test_database_names_to_dump_passes_through_name(): @@ -94,12 +128,10 @@ def test_database_names_to_dump_queries_mariadb_for_database_names(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive( - 'parse_extra_options' - ).and_return((), None) - flexmock(module).should_receive( - 'make_defaults_file_options' - ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ( 'mariadb', @@ -274,12 +306,12 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured def test_database_names_to_dump_runs_mariadb_with_list_options(): database = {'name': 'all', 'list_options': '--defaults-extra-file=mariadb.cnf --skip-ssl'} - flexmock(module).should_receive( - 'parse_extra_options' - ).and_return(('--skip-ssl',), 'mariadb.cnf') - flexmock(module).should_receive( - 'make_defaults_file_options' - ).with_args('root', 'trustsome1', 'mariadb.cnf').and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module).should_receive('parse_extra_options').and_return( + ('--skip-ssl',), 'mariadb.cnf' + ) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', 'mariadb.cnf' + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ( 'mariadb', @@ -305,12 +337,12 @@ def test_database_names_to_dump_runs_non_default_mariadb_with_list_options(): 'list_options': '--defaults-extra-file=mariadb.cnf --skip-ssl', 'mariadb_command': 'custom_mariadb', } - flexmock(module).should_receive( - 'parse_extra_options' - ).and_return(('--skip-ssl',), 'mariadb.cnf') - flexmock(module).should_receive( - 'make_defaults_file_options' - ).with_args('root', 'trustsome1', 'mariadb.cnf').and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module).should_receive('parse_extra_options').and_return( + ('--skip-ssl',), 'mariadb.cnf' + ) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', 'mariadb.cnf' + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module).should_receive('execute_command_and_capture_output').with_args( environment=None, full_command=( @@ -337,12 +369,10 @@ def test_execute_dump_command_runs_mariadb_dump(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive( - 'parse_extra_options' - ).and_return((), None) - flexmock(module).should_receive( - 'make_defaults_file_options' - ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -382,12 +412,10 @@ def test_execute_dump_command_runs_mariadb_dump_without_add_drop_database(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive( - 'parse_extra_options' - ).and_return((), None) - flexmock(module).should_receive( - 'make_defaults_file_options' - ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -426,12 +454,10 @@ def test_execute_dump_command_runs_mariadb_dump_with_hostname_and_port(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive( - 'parse_extra_options' - ).and_return((), None) - flexmock(module).should_receive( - 'make_defaults_file_options' - ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -477,12 +503,10 @@ def test_execute_dump_command_runs_mariadb_dump_with_username_and_password(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive( - 'parse_extra_options' - ).and_return((), None) - flexmock(module).should_receive( - 'make_defaults_file_options' - ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -522,12 +546,10 @@ def test_execute_dump_command_runs_mariadb_dump_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive( - 'parse_extra_options' - ).and_return(('--stuff=such',), None) - flexmock(module).should_receive( - 'make_defaults_file_options' - ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module).should_receive('parse_extra_options').and_return(('--stuff=such',), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -568,12 +590,10 @@ def test_execute_dump_command_runs_non_default_mariadb_dump_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive( - 'parse_extra_options' - ).and_return(('--stuff=such',), None) - flexmock(module).should_receive( - 'make_defaults_file_options' - ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module).should_receive('parse_extra_options').and_return(('--stuff=such',), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -614,12 +634,10 @@ def test_execute_dump_command_runs_non_default_mariadb_dump_with_options(): def test_execute_dump_command_with_duplicate_dump_skips_mariadb_dump(): flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(True) - flexmock(module).should_receive( - 'parse_extra_options' - ).and_return((), None) - flexmock(module).should_receive( - 'make_defaults_file_options' - ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() @@ -645,12 +663,10 @@ def test_execute_dump_command_with_dry_run_skips_mariadb_dump(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive( - 'parse_extra_options' - ).and_return((), None) - flexmock(module).should_receive( - 'make_defaults_file_options' - ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').never() @@ -726,12 +742,10 @@ def test_restore_data_source_dump_runs_mariadb_to_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive( - 'parse_extra_options' - ).and_return((), None) - flexmock(module).should_receive( - 'make_defaults_file_options' - ).with_args(None, None, None).and_return(()) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + None, None, None + ).and_return(()) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mariadb', '--batch'), @@ -764,12 +778,10 @@ def test_restore_data_source_dump_runs_mariadb_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive( - 'parse_extra_options' - ).and_return(('--harder',), None) - flexmock(module).should_receive( - 'make_defaults_file_options' - ).with_args(None, None, None).and_return(()) + flexmock(module).should_receive('parse_extra_options').and_return(('--harder',), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + None, None, None + ).and_return(()) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mariadb', '--harder', '--batch'), @@ -804,12 +816,10 @@ def test_restore_data_source_dump_runs_non_default_mariadb_with_options(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive( - 'parse_extra_options' - ).and_return(('--harder',), None) - flexmock(module).should_receive( - 'make_defaults_file_options' - ).with_args(None, None, None).and_return(()) + flexmock(module).should_receive('parse_extra_options').and_return(('--harder',), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + None, None, None + ).and_return(()) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('custom_mariadb', '--harder', '--batch'), @@ -842,12 +852,10 @@ def test_restore_data_source_dump_runs_mariadb_with_hostname_and_port(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive( - 'parse_extra_options' - ).and_return((), None) - flexmock(module).should_receive( - 'make_defaults_file_options' - ).with_args(None, None, None).and_return(()) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + None, None, None + ).and_return(()) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( @@ -889,12 +897,10 @@ def test_restore_data_source_dump_runs_mariadb_with_username_and_password(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive( - 'parse_extra_options' - ).and_return((), None) - flexmock(module).should_receive( - 'make_defaults_file_options' - ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mariadb', '--defaults-extra-file=/dev/fd/99', '--batch'), @@ -937,14 +943,10 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive( - 'parse_extra_options' - ).and_return((), None) - flexmock(module).should_receive( - 'make_defaults_file_options' - ).with_args('cliusername', 'clipassword', None).and_return( - ('--defaults-extra-file=/dev/fd/99',) - ) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'cliusername', 'clipassword', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( @@ -999,14 +1001,10 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive( - 'parse_extra_options' - ).and_return((), None) - flexmock(module).should_receive( - 'make_defaults_file_options' - ).with_args('restoreuser', 'restorepass', None).and_return( - ('--defaults-extra-file=/dev/fd/99',) - ) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'restoreuser', 'restorepass', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( @@ -1048,12 +1046,10 @@ def test_restore_data_source_dump_with_dry_run_skips_restore(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) - flexmock(module).should_receive( - 'parse_extra_options' - ).and_return((), None) - flexmock(module).should_receive( - 'make_defaults_file_options' - ).with_args(None, None, None).and_return(()) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + None, None, None + ).and_return(()) flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').never() From 839862cff05773c917fab20b46db51a8e8833129 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 28 Feb 2025 19:31:22 -0800 Subject: [PATCH 065/226] Update documentation link text about providing database passwords from external sources (#1009). --- docs/how-to/backup-your-databases.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index 769d4dbf..30a3590f 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -309,10 +309,8 @@ hooks: ### External passwords If you don't want to keep your database passwords in your borgmatic -configuration file, you can instead pass them in via [environment -variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/) -or command-line [configuration -overrides](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides). +configuration file, you can instead pass them in [from external credential +sources](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/). ### Configuration backups From b97372adf24a4913ab1c201bc0115691528f57d4 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 1 Mar 2025 08:49:42 -0800 Subject: [PATCH 066/226] Add MariaDB and MySQL anonymous pipe to NEWS (#1009). --- NEWS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS b/NEWS index 0d135f3b..fe3d6cf1 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,8 @@ 1.9.13.dev0 * #1001: Fix a ZFS error during snapshot cleanup. * #1003: In the Zabbix monitoring hook, support Zabbix 7.2's authentication changes. + * #1009: Send database passwords to MariaDB and MySQL via anonymous pipe, which is more secure than + using an environment variable. * Add a "verify_tls" option to the Uptime Kuma monitoring hook for disabling TLS verification. 1.9.12 From 25b6a49df726dea0e4db34cabe9a07bd45e6036d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 1 Mar 2025 10:04:04 -0800 Subject: [PATCH 067/226] Send database passwords to MongoDB via anonymous pipe (#1013). --- NEWS | 2 + borgmatic/hooks/data_source/mongodb.py | 39 +++++++++++------ tests/unit/hooks/data_source/test_mongodb.py | 44 ++++++++++++++++---- 3 files changed, 64 insertions(+), 21 deletions(-) diff --git a/NEWS b/NEWS index fe3d6cf1..5e48a5e2 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,8 @@ * #1003: In the Zabbix monitoring hook, support Zabbix 7.2's authentication changes. * #1009: Send database passwords to MariaDB and MySQL via anonymous pipe, which is more secure than using an environment variable. + * #1013: Send database passwords to MongoDB via anonymous pipe, which is more secure than using + "--password" on the command-line. * Add a "verify_tls" option to the Uptime Kuma monitoring hook for disabling TLS verification. 1.9.12 diff --git a/borgmatic/hooks/data_source/mongodb.py b/borgmatic/hooks/data_source/mongodb.py index 59811101..37cc63dd 100644 --- a/borgmatic/hooks/data_source/mongodb.py +++ b/borgmatic/hooks/data_source/mongodb.py @@ -89,12 +89,36 @@ def dump_data_sources( return processes +def make_password_config_file(password): + ''' + Given a database password, write it as a MongoDB configuration file to an anonymous pipe and + return its filename. The idea is that this is a more secure way to transmit a password to + MongoDB than providing it directly on the command-line. + + Do not use the returned value for multiple different command invocations. That will not work + because each pipe is "used up" once read. + ''' + logger.debug('Writing MongoDB password to configuration file pipe') + + read_file_descriptor, write_file_descriptor = os.pipe() + os.write(write_file_descriptor, f'password: {password}'.encode('utf-8')) + os.close(write_file_descriptor) + + # This plus subprocess.Popen(..., close_fds=False) in execute.py is necessary for the database + # client child process to inherit the file descriptor. + os.set_inheritable(read_file_descriptor, True) + + return f'/dev/fd/{read_file_descriptor}' + + def build_dump_command(database, config, dump_filename, dump_format): ''' Return the mongodump command from a single database configuration. ''' all_databases = database['name'] == 'all' + password = borgmatic.hooks.credential.parse.resolve_credential(database.get('password'), config) + return ( ('mongodump',) + (('--out', shlex.quote(dump_filename)) if dump_format == 'directory' else ()) @@ -112,18 +136,7 @@ def build_dump_command(database, config, dump_filename, dump_format): if 'username' in database else () ) - + ( - ( - '--password', - shlex.quote( - borgmatic.hooks.credential.parse.resolve_credential( - database['password'], config - ) - ), - ) - if 'password' in database - else () - ) + + (('--config', make_password_config_file(password)) if password else ()) + ( ('--authenticationDatabase', shlex.quote(database['authentication_database'])) if 'authentication_database' in database @@ -251,7 +264,7 @@ def build_restore_command(extract_process, database, config, dump_filename, conn if username: command.extend(('--username', username)) if password: - command.extend(('--password', password)) + command.extend(('--config', make_password_config_file(password))) if 'authentication_database' in database: command.extend(('--authenticationDatabase', database['authentication_database'])) if 'restore_options' in database: diff --git a/tests/unit/hooks/data_source/test_mongodb.py b/tests/unit/hooks/data_source/test_mongodb.py index ea22e824..0cef73f8 100644 --- a/tests/unit/hooks/data_source/test_mongodb.py +++ b/tests/unit/hooks/data_source/test_mongodb.py @@ -127,6 +127,9 @@ def test_dump_data_sources_runs_mongodump_with_username_and_password(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_password_config_file').with_args('trustsome1').and_return( + '/dev/fd/99' + ) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -134,8 +137,8 @@ def test_dump_data_sources_runs_mongodump_with_username_and_password(): 'mongodump', '--username', 'mongo', - '--password', - 'trustsome1', + '--config', + '/dev/fd/99', '--authenticationDatabase', 'admin', '--db', @@ -243,6 +246,22 @@ def test_dump_data_sources_runs_mongodumpall_for_all_databases(): ) == [process] +def test_make_password_config_file_writes_password_to_pipe(): + read_file_descriptor = 99 + write_file_descriptor = flexmock() + + flexmock(module.os).should_receive('pipe').and_return( + (read_file_descriptor, write_file_descriptor) + ) + flexmock(module.os).should_receive('write').with_args( + write_file_descriptor, b'password: trustsome1' + ).once() + flexmock(module.os).should_receive('close') + flexmock(module.os).should_receive('set_inheritable') + + assert module.make_password_config_file('trustsome1') == '/dev/fd/99' + + def test_build_dump_command_with_username_injection_attack_gets_escaped(): database = {'name': 'test', 'username': 'bob; naughty-command'} flexmock(module.borgmatic.hooks.credential.parse).should_receive( @@ -345,6 +364,9 @@ def test_restore_data_source_dump_runs_mongorestore_with_username_and_password() flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_password_config_file').with_args('trustsome1').and_return( + '/dev/fd/99' + ) flexmock(module).should_receive('execute_command_with_processes').with_args( [ 'mongorestore', @@ -352,8 +374,8 @@ def test_restore_data_source_dump_runs_mongorestore_with_username_and_password() '--drop', '--username', 'mongo', - '--password', - 'trustsome1', + '--config', + '/dev/fd/99', '--authenticationDatabase', 'admin', ], @@ -399,6 +421,9 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_password_config_file').with_args( + 'clipassword' + ).and_return('/dev/fd/99') flexmock(module).should_receive('execute_command_with_processes').with_args( [ 'mongorestore', @@ -410,8 +435,8 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ 'cliport', '--username', 'cliusername', - '--password', - 'clipassword', + '--config', + '/dev/fd/99', '--authenticationDatabase', 'admin', ], @@ -457,6 +482,9 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_password_config_file').with_args( + 'restorepass' + ).and_return('/dev/fd/99') flexmock(module).should_receive('execute_command_with_processes').with_args( [ 'mongorestore', @@ -468,8 +496,8 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ 'restoreport', '--username', 'restoreuser', - '--password', - 'restorepass', + '--config', + '/dev/fd/99', '--authenticationDatabase', 'admin', ], From 54afe87a9f3e50c7f247d7bb4bbd1c03f87b103f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 1 Mar 2025 17:29:16 -0800 Subject: [PATCH 068/226] Add a "compression" option to the PostgreSQL database hook (#975). --- NEWS | 1 + borgmatic/config/schema.yaml | 11 +++ borgmatic/hooks/data_source/postgresql.py | 2 + .../unit/hooks/data_source/test_postgresql.py | 94 +++++++++++++++++++ 4 files changed, 108 insertions(+) diff --git a/NEWS b/NEWS index 5e48a5e2..ac0e36b7 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ 1.9.13.dev0 + * #975: Add a "compression" option to the PostgreSQL database hook. * #1001: Fix a ZFS error during snapshot cleanup. * #1003: In the Zabbix monitoring hook, support Zabbix 7.2's authentication changes. * #1009: Send database passwords to MariaDB and MySQL via anonymous pipe, which is more secure than diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 33f39a70..109b1a8f 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1040,6 +1040,17 @@ properties: individual databases. See the pg_dump documentation for more about formats. example: directory + compression: + type: ["string", "integer"] + description: | + Database dump compression level (integer) or method + ("gzip", "lz4", "zstd", or "none") and optional + colon-separated detail. Defaults to moderate "gzip" for + "custom" and "directory" formats and no compression for + the "plain" format. Compression is not supported for the + "tar" format. Be aware that Borg does its own + compression as well, so you may not need it in both + places. ssl_mode: type: string enum: ['disable', 'allow', 'prefer', diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index 559cc4b8..4f470c52 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -159,6 +159,7 @@ def dump_data_sources( for database_name in dump_database_names: dump_format = database.get('format', None if database_name == 'all' else 'custom') + compression = database.get('compression') default_dump_command = 'pg_dumpall' if database_name == 'all' else 'pg_dump' dump_command = tuple( shlex.quote(part) @@ -199,6 +200,7 @@ def dump_data_sources( ) + (('--no-owner',) if database.get('no_owner', False) else ()) + (('--format', shlex.quote(dump_format)) if dump_format else ()) + + (('--compress', shlex.quote(str(compression))) if compression is not None else ()) + (('--file', shlex.quote(dump_filename)) if dump_format == 'directory' else ()) + ( tuple(shlex.quote(option) for option in database['options'].split(' ')) diff --git a/tests/unit/hooks/data_source/test_postgresql.py b/tests/unit/hooks/data_source/test_postgresql.py index 1037cf66..f1589a6e 100644 --- a/tests/unit/hooks/data_source/test_postgresql.py +++ b/tests/unit/hooks/data_source/test_postgresql.py @@ -555,6 +555,100 @@ def test_dump_data_sources_runs_pg_dump_with_directory_format(): ) +def test_dump_data_sources_runs_pg_dump_with_string_compression(): + databases = [{'name': 'foo', 'compression': 'winrar'}] + processes = [flexmock()] + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)) + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( + 'databases/localhost/foo' + ) + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'pg_dump', + '--no-password', + '--clean', + '--if-exists', + '--format', + 'custom', + '--compress', + 'winrar', + 'foo', + '>', + 'databases/localhost/foo', + ), + shell=True, + environment={'PGSSLMODE': 'disable'}, + run_to_completion=False, + ).and_return(processes[0]).once() + + assert ( + module.dump_data_sources( + databases, + {}, + config_paths=('test.yaml',), + borgmatic_runtime_directory='/run/borgmatic', + patterns=[], + dry_run=False, + ) + == processes + ) + + +def test_dump_data_sources_runs_pg_dump_with_integer_compression(): + databases = [{'name': 'foo', 'compression': 0}] + processes = [flexmock()] + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)) + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( + 'databases/localhost/foo' + ) + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'pg_dump', + '--no-password', + '--clean', + '--if-exists', + '--format', + 'custom', + '--compress', + '0', + 'foo', + '>', + 'databases/localhost/foo', + ), + shell=True, + environment={'PGSSLMODE': 'disable'}, + run_to_completion=False, + ).and_return(processes[0]).once() + + assert ( + module.dump_data_sources( + databases, + {}, + config_paths=('test.yaml',), + borgmatic_runtime_directory='/run/borgmatic', + patterns=[], + dry_run=False, + ) + == processes + ) + + def test_dump_data_sources_runs_pg_dump_with_options(): databases = [{'name': 'foo', 'options': '--stuff=such'}] process = flexmock() From 114f5702b281d7d894ce45b71ac52a902d2cf358 Mon Sep 17 00:00:00 2001 From: lingfish Date: Sun, 2 Mar 2025 14:22:57 +1100 Subject: [PATCH 069/226] Expand a little on the specifics of backups of an LVM volume. --- docs/how-to/snapshot-your-filesystems.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/how-to/snapshot-your-filesystems.md b/docs/how-to/snapshot-your-filesystems.md index f1b6f8e2..1041dc05 100644 --- a/docs/how-to/snapshot-your-filesystems.md +++ b/docs/how-to/snapshot-your-filesystems.md @@ -205,6 +205,14 @@ Volume Manager) and sending those snapshots to Borg for backup. LVM isn't itself a filesystem, but it can take snapshots at the layer right below your filesystem. +Note that, due to borg being a file-level backup, this feature is really only +suitable for filesystems, not whole disk or raw images containing multiple +filesystems (for example, if you're using a LVM volume to run a Windows +KVM that contains an MBR, partitions, etc.) + +In those cases, you can omit the `lvm:` option, and use borg's own support for +[image backup](https://borgbackup.readthedocs.io/en/stable/deployment/image-backup.html). + To use this feature, first you need one or more mounted LVM logical volumes. Then, enable LVM within borgmatic by adding the following line to your configuration file: From 024006f4c03a3b31c221f106effc59202e3b9e5e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 1 Mar 2025 20:56:40 -0800 Subject: [PATCH 070/226] Title case Borg. --- docs/how-to/snapshot-your-filesystems.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/how-to/snapshot-your-filesystems.md b/docs/how-to/snapshot-your-filesystems.md index 1041dc05..bd569957 100644 --- a/docs/how-to/snapshot-your-filesystems.md +++ b/docs/how-to/snapshot-your-filesystems.md @@ -205,12 +205,12 @@ Volume Manager) and sending those snapshots to Borg for backup. LVM isn't itself a filesystem, but it can take snapshots at the layer right below your filesystem. -Note that, due to borg being a file-level backup, this feature is really only +Note that, due to Borg being a file-level backup, this feature is really only suitable for filesystems, not whole disk or raw images containing multiple filesystems (for example, if you're using a LVM volume to run a Windows -KVM that contains an MBR, partitions, etc.) +KVM that contains an MBR, partitions, etc.). -In those cases, you can omit the `lvm:` option, and use borg's own support for +In those cases, you can omit the `lvm:` option and use Borg's own support for [image backup](https://borgbackup.readthedocs.io/en/stable/deployment/image-backup.html). To use this feature, first you need one or more mounted LVM logical volumes. From 4b2f7e03af8260314a1d579fbd7710dbedc7b27e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 1 Mar 2025 21:02:32 -0800 Subject: [PATCH 071/226] Fix broken "config generate" (#975). --- borgmatic/config/generate.py | 1 + borgmatic/config/schema.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index f42c1690..6b375b76 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -41,6 +41,7 @@ def schema_to_sample_configuration(schema, level=0, parent_is_sequence=False): ''' schema_type = schema.get('type') example = schema.get('example') + if example is not None: return example diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 109b1a8f..23db735d 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1051,6 +1051,7 @@ properties: "tar" format. Be aware that Borg does its own compression as well, so you may not need it in both places. + example: none ssl_mode: type: string enum: ['disable', 'allow', 'prefer', From 2a16ffab1be5c4dc7bd0bba83f6b93997f955f69 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 2 Mar 2025 10:32:57 -0800 Subject: [PATCH 072/226] When ctrl-C is pressed, ensure Borg actually exits (#1015). --- NEWS | 1 + borgmatic/signals.py | 3 +++ tests/unit/test_signals.py | 8 +++++--- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index ac0e36b7..3f64c8d2 100644 --- a/NEWS +++ b/NEWS @@ -6,6 +6,7 @@ using an environment variable. * #1013: Send database passwords to MongoDB via anonymous pipe, which is more secure than using "--password" on the command-line. + * #1015: When ctrl-C is pressed, ensure Borg actually exits. * Add a "verify_tls" option to the Uptime Kuma monitoring hook for disabling TLS verification. 1.9.12 diff --git a/borgmatic/signals.py b/borgmatic/signals.py index 7ec31d91..40fa5295 100644 --- a/borgmatic/signals.py +++ b/borgmatic/signals.py @@ -24,6 +24,9 @@ def handle_signal(signal_number, frame): logger.critical('Exiting due to TERM signal') sys.exit(EXIT_CODE_FROM_SIGNAL + signal.SIGTERM) elif signal_number == signal.SIGINT: + # Borg doesn't always exit on a SIGINT, so give it a little encouragement. + os.killpg(os.getpgrp(), signal.SIGTERM) + raise KeyboardInterrupt() diff --git a/tests/unit/test_signals.py b/tests/unit/test_signals.py index 86bb759c..57ee14ab 100644 --- a/tests/unit/test_signals.py +++ b/tests/unit/test_signals.py @@ -26,7 +26,7 @@ def test_handle_signal_bails_on_recursion(): def test_handle_signal_exits_on_sigterm(): signal_number = module.signal.SIGTERM frame = flexmock(f_back=flexmock(f_code=flexmock(co_name='something'))) - flexmock(module.os).should_receive('getpgrp').and_return(flexmock) + flexmock(module.os).should_receive('getpgrp').and_return(flexmock()) flexmock(module.os).should_receive('killpg') flexmock(module.sys).should_receive('exit').with_args( module.EXIT_CODE_FROM_SIGNAL + signal_number @@ -38,8 +38,10 @@ def test_handle_signal_exits_on_sigterm(): def test_handle_signal_raises_on_sigint(): signal_number = module.signal.SIGINT frame = flexmock(f_back=flexmock(f_code=flexmock(co_name='something'))) - flexmock(module.os).should_receive('getpgrp').and_return(flexmock) - flexmock(module.os).should_receive('killpg') + process_group = flexmock() + flexmock(module.os).should_receive('getpgrp').and_return(process_group) + flexmock(module.os).should_receive('killpg').with_args(process_group, module.signal.SIGINT) + flexmock(module.os).should_receive('killpg').with_args(process_group, module.signal.SIGTERM) flexmock(module.sys).should_receive('exit').never() with pytest.raises(KeyboardInterrupt): From 1f86100f26aeda612b1523fab04479b8314de125 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 2 Mar 2025 20:10:20 -0800 Subject: [PATCH 073/226] NEWS wording tweaks. --- NEWS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 3f64c8d2..b6fe702e 100644 --- a/NEWS +++ b/NEWS @@ -5,8 +5,8 @@ * #1009: Send database passwords to MariaDB and MySQL via anonymous pipe, which is more secure than using an environment variable. * #1013: Send database passwords to MongoDB via anonymous pipe, which is more secure than using - "--password" on the command-line. - * #1015: When ctrl-C is pressed, ensure Borg actually exits. + "--password" on the command-line! + * #1015: When ctrl-C is pressed, more strongly encourage Borg to actually exit. * Add a "verify_tls" option to the Uptime Kuma monitoring hook for disabling TLS verification. 1.9.12 From 55c0ab16102d50396af0f0b510dc3307bcc2f4f9 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 3 Mar 2025 10:07:03 -0800 Subject: [PATCH 074/226] Add "tls" options to the MariaDB and MySQL database hooks. --- NEWS | 2 + borgmatic/config/schema.yaml | 28 +++ borgmatic/hooks/data_source/mariadb.py | 7 + borgmatic/hooks/data_source/mysql.py | 7 + tests/unit/hooks/data_source/test_mariadb.py | 229 ++++++++++++++++++ tests/unit/hooks/data_source/test_mysql.py | 241 +++++++++++++++++++ 6 files changed, 514 insertions(+) diff --git a/NEWS b/NEWS index b6fe702e..c9249dff 100644 --- a/NEWS +++ b/NEWS @@ -8,6 +8,8 @@ "--password" on the command-line! * #1015: When ctrl-C is pressed, more strongly encourage Borg to actually exit. * Add a "verify_tls" option to the Uptime Kuma monitoring hook for disabling TLS verification. + * Add "tls" options to the MariaDB and MySQL database hooks to enable or disable TLS encryption + between client and server. 1.9.12 * #1005: Fix the credential hooks to avoid using Python 3.12+ string features. Now borgmatic will diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 23db735d..de8dec6f 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1210,6 +1210,20 @@ properties: Defaults to the "password" option. Supports the "{credential ...}" syntax. example: trustsome1 + tls: + type: boolean + description: | + Whether to TLS-encrypt data transmitted between the + client and server. The default varies based on the + MariaDB version. + example: false + restore_tls: + type: boolean + description: | + Whether to TLS-encrypt data transmitted between the + client and restore server. The default varies based on + the MariaDB version. + example: false mariadb_dump_command: type: string description: | @@ -1340,6 +1354,20 @@ properties: Defaults to the "password" option. Supports the "{credential ...}" syntax. example: trustsome1 + tls: + type: boolean + description: | + Whether to TLS-encrypt data transmitted between the + client and server. The default varies based on the + MySQL installation. + example: false + restore_tls: + type: boolean + description: | + Whether to TLS-encrypt data transmitted between the + client and restore server. The default varies based on + the MySQL installation. + example: false mysql_dump_command: type: string description: | diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index fc879142..cb05f5b9 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -125,6 +125,8 @@ def database_names_to_dump(database, config, username, password, environment, dr + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + + (('--ssl',) if database.get('tls') is True else ()) + + (('--skip-ssl',) if database.get('tls') is False else ()) + ('--skip-column-names', '--batch') + ('--execute', 'show schemas') ) @@ -188,6 +190,8 @@ def execute_dump_command( + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + + (('--ssl',) if database.get('tls') is True else ()) + + (('--skip-ssl',) if database.get('tls') is False else ()) + ('--databases',) + database_names + ('--result-file', dump_filename) @@ -357,6 +361,7 @@ def restore_data_source_dump( port = str( connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')) ) + tls = data_source.get('restore_tls', data_source.get('tls')) username = borgmatic.hooks.credential.parse.resolve_credential( ( connection_params['username'] @@ -384,6 +389,8 @@ def restore_data_source_dump( + (('--host', hostname) if hostname else ()) + (('--port', str(port)) if port else ()) + (('--protocol', 'tcp') if hostname or port else ()) + + (('--ssl',) if tls is True else ()) + + (('--skip-ssl',) if tls is False else ()) ) environment = dict(os.environ) diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index 13d77041..7c5f84f2 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -54,6 +54,8 @@ def database_names_to_dump(database, config, username, password, environment, dr + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + + (('--ssl',) if database.get('tls') is True else ()) + + (('--skip-ssl',) if database.get('tls') is False else ()) + ('--skip-column-names', '--batch') + ('--execute', 'show schemas') ) @@ -117,6 +119,8 @@ def execute_dump_command( + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) + + (('--ssl',) if database.get('tls') is True else ()) + + (('--skip-ssl',) if database.get('tls') is False else ()) + ('--databases',) + database_names + ('--result-file', dump_filename) @@ -286,6 +290,7 @@ def restore_data_source_dump( port = str( connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')) ) + tls = data_source.get('restore_tls', data_source.get('tls')) username = borgmatic.hooks.credential.parse.resolve_credential( ( connection_params['username'] @@ -317,6 +322,8 @@ def restore_data_source_dump( + (('--host', hostname) if hostname else ()) + (('--port', str(port)) if port else ()) + (('--protocol', 'tcp') if hostname or port else ()) + + (('--ssl',) if tls is True else ()) + + (('--skip-ssl',) if tls is False else ()) ) environment = dict(os.environ) diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py index fe183865..c6b88099 100644 --- a/tests/unit/hooks/data_source/test_mariadb.py +++ b/tests/unit/hooks/data_source/test_mariadb.py @@ -151,6 +151,64 @@ def test_database_names_to_dump_queries_mariadb_for_database_names(): assert names == ('foo', 'bar') +def test_database_names_to_dump_runs_mariadb_with_tls(): + environment = flexmock() + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + ( + 'mariadb', + '--defaults-extra-file=/dev/fd/99', + '--ssl', + '--skip-column-names', + '--batch', + '--execute', + 'show schemas', + ), + environment=environment, + ).and_return('foo\nbar\nmysql\n').once() + + names = module.database_names_to_dump( + {'name': 'all', 'tls': True}, {}, 'root', 'trustsome1', environment, dry_run=False + ) + + assert names == ('foo', 'bar') + + +def test_database_names_to_dump_runs_mariadb_without_tls(): + environment = flexmock() + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + ( + 'mariadb', + '--defaults-extra-file=/dev/fd/99', + '--skip-ssl', + '--skip-column-names', + '--batch', + '--execute', + 'show schemas', + ), + environment=environment, + ).and_return('foo\nbar\nmysql\n').once() + + names = module.database_names_to_dump( + {'name': 'all', 'tls': False}, {}, 'root', 'trustsome1', environment, dry_run=False + ) + + assert names == ('foo', 'bar') + + def test_use_streaming_true_for_any_databases(): assert module.use_streaming( databases=[flexmock(), flexmock()], @@ -496,6 +554,94 @@ def test_execute_dump_command_runs_mariadb_dump_with_hostname_and_port(): ) +def test_execute_dump_command_runs_mariadb_dump_with_tls(): + process = flexmock() + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'mariadb-dump', + '--defaults-extra-file=/dev/fd/99', + '--add-drop-database', + '--ssl', + '--databases', + 'foo', + '--result-file', + 'dump', + ), + environment=None, + run_to_completion=False, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={'name': 'foo', 'tls': True}, + config={}, + username='root', + password='trustsome1', + dump_path=flexmock(), + database_names=('foo',), + environment=None, + dry_run=False, + dry_run_label='', + ) + == process + ) + + +def test_execute_dump_command_runs_mariadb_dump_without_tls(): + process = flexmock() + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'mariadb-dump', + '--defaults-extra-file=/dev/fd/99', + '--add-drop-database', + '--skip-ssl', + '--databases', + 'foo', + '--result-file', + 'dump', + ), + environment=None, + run_to_completion=False, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={'name': 'foo', 'tls': False}, + config={}, + username='root', + password='trustsome1', + dump_path=flexmock(), + database_names=('foo',), + environment=None, + dry_run=False, + dry_run_label='', + ) + == process + ) + + def test_execute_dump_command_runs_mariadb_dump_with_username_and_password(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') @@ -890,6 +1036,86 @@ def test_restore_data_source_dump_runs_mariadb_with_hostname_and_port(): ) +def test_restore_data_source_dump_runs_mariadb_with_tls(): + hook_config = [{'name': 'foo', 'tls': True}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + None, None, None + ).and_return(()) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'mariadb', + '--batch', + '--ssl', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + environment={'USER': 'root'}, + ).once() + + module.restore_data_source_dump( + hook_config, + {}, + data_source=hook_config[0], + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + borgmatic_runtime_directory='/run/borgmatic', + ) + + +def test_restore_data_source_dump_runs_mariadb_without_tls(): + hook_config = [{'name': 'foo', 'tls': False}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + None, None, None + ).and_return(()) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'mariadb', + '--batch', + '--skip-ssl', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + environment={'USER': 'root'}, + ).once() + + module.restore_data_source_dump( + hook_config, + {}, + data_source=hook_config[0], + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + borgmatic_runtime_directory='/run/borgmatic', + ) + + def test_restore_data_source_dump_runs_mariadb_with_username_and_password(): hook_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}] extract_process = flexmock(stdout=flexmock()) @@ -990,10 +1216,12 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ 'password': 'trustsome1', 'hostname': 'dbhost', 'port': 'dbport', + 'tls': True, 'restore_username': 'restoreuser', 'restore_password': 'restorepass', 'restore_hostname': 'restorehost', 'restore_port': 'restoreport', + 'restore_tls': False, } ] extract_process = flexmock(stdout=flexmock()) @@ -1017,6 +1245,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ 'restoreport', '--protocol', 'tcp', + '--skip-ssl', ), processes=[extract_process], output_log_level=logging.DEBUG, diff --git a/tests/unit/hooks/data_source/test_mysql.py b/tests/unit/hooks/data_source/test_mysql.py index 2e629dc3..63bb7a72 100644 --- a/tests/unit/hooks/data_source/test_mysql.py +++ b/tests/unit/hooks/data_source/test_mysql.py @@ -60,6 +60,68 @@ def test_database_names_to_dump_queries_mysql_for_database_names(): assert names == ('foo', 'bar') +def test_database_names_to_dump_runs_mysql_with_tls(): + environment = flexmock() + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + ( + 'mysql', + '--defaults-extra-file=/dev/fd/99', + '--ssl', + '--skip-column-names', + '--batch', + '--execute', + 'show schemas', + ), + environment=environment, + ).and_return('foo\nbar\nmysql\n').once() + + names = module.database_names_to_dump( + {'name': 'all', 'tls': True}, {}, 'root', 'trustsome1', environment, dry_run=False + ) + + assert names == ('foo', 'bar') + + +def test_database_names_to_dump_runs_mysql_without_tls(): + environment = flexmock() + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + ( + 'mysql', + '--defaults-extra-file=/dev/fd/99', + '--skip-ssl', + '--skip-column-names', + '--batch', + '--execute', + 'show schemas', + ), + environment=environment, + ).and_return('foo\nbar\nmysql\n').once() + + names = module.database_names_to_dump( + {'name': 'all', 'tls': False}, {}, 'root', 'trustsome1', environment, dry_run=False + ) + + assert names == ('foo', 'bar') + + def test_use_streaming_true_for_any_databases(): assert module.use_streaming( databases=[flexmock(), flexmock()], @@ -408,6 +470,98 @@ def test_execute_dump_command_runs_mysqldump_with_hostname_and_port(): ) +def test_execute_dump_command_runs_mysqldump_with_tls(): + process = flexmock() + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'mysqldump', + '--defaults-extra-file=/dev/fd/99', + '--add-drop-database', + '--ssl', + '--databases', + 'foo', + '--result-file', + 'dump', + ), + environment=None, + run_to_completion=False, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={'name': 'foo', 'tls': True}, + config={}, + username='root', + password='trustsome1', + dump_path=flexmock(), + database_names=('foo',), + environment=None, + dry_run=False, + dry_run_label='', + ) + == process + ) + + +def test_execute_dump_command_runs_mysqldump_without_tls(): + process = flexmock() + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'mysqldump', + '--defaults-extra-file=/dev/fd/99', + '--add-drop-database', + '--skip-ssl', + '--databases', + 'foo', + '--result-file', + 'dump', + ), + environment=None, + run_to_completion=False, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={'name': 'foo', 'tls': False}, + config={}, + username='root', + password='trustsome1', + dump_path=flexmock(), + database_names=('foo',), + environment=None, + dry_run=False, + dry_run_label='', + ) + == process + ) + + def test_execute_dump_command_runs_mysqldump_with_username_and_password(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') @@ -816,6 +970,90 @@ def test_restore_data_source_dump_runs_mysql_with_hostname_and_port(): ) +def test_restore_data_source_dump_runs_mysql_with_tls(): + hook_config = [{'name': 'foo', 'tls': True}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'mysql', + '--batch', + '--ssl', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + environment={'USER': 'root'}, + ).once() + + module.restore_data_source_dump( + hook_config, + {}, + data_source=hook_config[0], + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + borgmatic_runtime_directory='/run/borgmatic', + ) + + +def test_restore_data_source_dump_runs_mysql_without_tls(): + hook_config = [{'name': 'foo', 'tls': False}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'mysql', + '--batch', + '--skip-ssl', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + environment={'USER': 'root'}, + ).once() + + module.restore_data_source_dump( + hook_config, + {}, + data_source=hook_config[0], + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + borgmatic_runtime_directory='/run/borgmatic', + ) + + def test_restore_data_source_dump_runs_mysql_with_username_and_password(): hook_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}] extract_process = flexmock(stdout=flexmock()) @@ -922,10 +1160,12 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ 'password': 'trustsome1', 'hostname': 'dbhost', 'port': 'dbport', + 'tls': True, 'restore_username': 'restoreuser', 'restore_password': 'restorepass', 'restore_hostname': 'restorehost', 'restore_port': 'restoreport', + 'restore_tls': False, } ] extract_process = flexmock(stdout=flexmock()) @@ -953,6 +1193,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ 'restoreport', '--protocol', 'tcp', + '--skip-ssl', ), processes=[extract_process], output_log_level=logging.DEBUG, From dbe82ff11e02da8121cbf1dc639fb55bed912ae2 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 3 Mar 2025 10:21:15 -0800 Subject: [PATCH 075/226] Bump version for release. --- NEWS | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index c9249dff..25fdc460 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -1.9.13.dev0 +1.9.13 * #975: Add a "compression" option to the PostgreSQL database hook. * #1001: Fix a ZFS error during snapshot cleanup. * #1003: In the Zabbix monitoring hook, support Zabbix 7.2's authentication changes. diff --git a/pyproject.toml b/pyproject.toml index 42c64dc2..54031a9c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "borgmatic" -version = "1.9.13.dev0" +version = "1.9.13" authors = [ { name="Dan Helfman", email="witten@torsion.org" }, ] From ddfd3c6ca1538662f4f46db79d209f398c5357d6 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 3 Mar 2025 16:02:22 -0800 Subject: [PATCH 076/226] Clarify Zabbix monitoring hook documentation about creating items (#936). --- NEWS | 3 +++ docs/how-to/monitor-your-backups.md | 16 ++++++++++++---- pyproject.toml | 2 +- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/NEWS b/NEWS index 25fdc460..e77e1003 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +1.9.14.dev0 + * #936: Clarify Zabbix monitoring hook documentation about creating items. + 1.9.13 * #975: Add a "compression" option to the PostgreSQL database hook. * #1001: Fix a ZFS error during snapshot cleanup. diff --git a/docs/how-to/monitor-your-backups.md b/docs/how-to/monitor-your-backups.md index 01649315..1f0c959e 100644 --- a/docs/how-to/monitor-your-backups.md +++ b/docs/how-to/monitor-your-backups.md @@ -724,11 +724,19 @@ Authentication can be accomplished via `api_key` or both `username` and ### Items -The item to be updated can be chosen by either declaring the `itemid` or both -`host` and `key`. If all three are declared, only `itemid` is used. +borgmatic writes its monitoring updates to a particular Zabbix item, which +you'll need to create in advance. In the Zabbix web UI, [make a new item with a +Type of "Zabbix +trapper"](https://www.zabbix.com/documentation/current/en/manual/config/items/itemtypes/trapper) +and a named Key. The "Type of information" for the item should be "Text", and +"History" designates how much data you want to retain. -Keep in mind that `host` is referring to the "Host name" on the Zabbix server -and not the "Visual name". +When configuring borgmatic with this item to be updated, you can either declare +the `itemid` or both `host` and `key`. If all three are declared, only `itemid` +is used. + +Keep in mind that `host` refers to the "Host name" on the Zabbix server and not +the "Visual name". ## Scripting borgmatic diff --git a/pyproject.toml b/pyproject.toml index 54031a9c..cd1ce7a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "borgmatic" -version = "1.9.13" +version = "1.9.14.dev0" authors = [ { name="Dan Helfman", email="witten@torsion.org" }, ] From c0135864c2b70eb7d76716a5d35da796c671b148 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 4 Mar 2025 08:55:09 -0800 Subject: [PATCH 077/226] With the PagerDuty monitoring hook, send borgmatic logs to PagerDuty so they show up in the incident UI (#409). --- NEWS | 3 ++ borgmatic/config/schema.yaml | 6 +++ borgmatic/hooks/monitoring/logs.py | 2 +- borgmatic/hooks/monitoring/pagerduty.py | 43 +++++++++++---- docs/how-to/monitor-your-backups.md | 21 ++++++++ tests/unit/hooks/monitoring/test_pagerduty.py | 52 +++++++++++++++++++ 6 files changed, 115 insertions(+), 12 deletions(-) diff --git a/NEWS b/NEWS index e77e1003..5a1bd387 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,7 @@ 1.9.14.dev0 + * #409: With the PagerDuty monitoring hook, send borgmatic logs to PagerDuty so they show up in the + incident UI. See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook * #936: Clarify Zabbix monitoring hook documentation about creating items. 1.9.13 diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index de8dec6f..f6ec574e 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -2279,6 +2279,12 @@ properties: PagerDuty integration key used to notify PagerDuty when a backup errors. Supports the "{credential ...}" syntax. example: a177cad45bd374409f78906a810a3074 + send_logs: + type: boolean + description: | + Send borgmatic logs to PagerDuty when a backup errors. + Defaults to true. + example: false description: | Configuration for a monitoring integration with PagerDuty. Create an account at https://www.pagerduty.com if you'd like to use this diff --git a/borgmatic/hooks/monitoring/logs.py b/borgmatic/hooks/monitoring/logs.py index de1bef60..a2e2f39f 100644 --- a/borgmatic/hooks/monitoring/logs.py +++ b/borgmatic/hooks/monitoring/logs.py @@ -64,7 +64,7 @@ def get_handler(identifier): def format_buffered_logs_for_payload(identifier): ''' Get the handler previously added to the root logger, and slurp buffered logs out of it to - send to Healthchecks. + send to the monitoring service. ''' try: buffering_handler = get_handler(identifier) diff --git a/borgmatic/hooks/monitoring/pagerduty.py b/borgmatic/hooks/monitoring/pagerduty.py index e238957b..d98375d9 100644 --- a/borgmatic/hooks/monitoring/pagerduty.py +++ b/borgmatic/hooks/monitoring/pagerduty.py @@ -6,20 +6,36 @@ import platform import requests import borgmatic.hooks.credential.parse +import borgmatic.hooks.monitoring.logs from borgmatic.hooks.monitoring import monitor logger = logging.getLogger(__name__) EVENTS_API_URL = 'https://events.pagerduty.com/v2/enqueue' +DEFAULT_LOGS_PAYLOAD_LIMIT_BYTES = 10000 +HANDLER_IDENTIFIER = 'pagerduty' -def initialize_monitor( - integration_key, config, config_filename, monitoring_log_level, dry_run -): # pragma: no cover +def initialize_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run): ''' - No initialization is necessary for this monitor. + Add a handler to the root logger that stores in memory the most recent logs emitted. That way, + we can send them all to PagerDuty upon a failure state. But skip this if the "send_logs" option + is false. ''' - pass + if hook_config.get('send_logs') is False: + return + + ping_body_limit = max( + DEFAULT_LOGS_PAYLOAD_LIMIT_BYTES + - len(borgmatic.hooks.monitoring.logs.PAYLOAD_TRUNCATION_INDICATOR), + 0, + ) + + borgmatic.hooks.monitoring.logs.add_handler( + borgmatic.hooks.monitoring.logs.Forgetful_buffering_handler( + HANDLER_IDENTIFIER, ping_body_limit, monitoring_log_level + ) + ) def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run): @@ -37,9 +53,6 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev dry_run_label = ' (dry run; not actually sending)' if dry_run else '' logger.info(f'Sending failure event to PagerDuty {dry_run_label}') - if dry_run: - return - try: integration_key = borgmatic.hooks.credential.parse.resolve_credential( hook_config.get('integration_key'), config @@ -48,6 +61,10 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev logger.warning(f'PagerDuty credential error: {error}') return + logs_payload = borgmatic.hooks.monitoring.logs.format_buffered_logs_for_payload( + HANDLER_IDENTIFIER + ) + hostname = platform.node() local_timestamp = datetime.datetime.now(datetime.timezone.utc).astimezone().isoformat() payload = json.dumps( @@ -66,11 +83,14 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev 'hostname': hostname, 'configuration filename': config_filename, 'server time': local_timestamp, + 'logs': logs_payload, }, }, } ) - logger.debug(f'Using PagerDuty payload: {payload}') + + if dry_run: + return logging.getLogger('urllib3').setLevel(logging.ERROR) try: @@ -83,6 +103,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev def destroy_monitor(ping_url_or_uuid, config, monitoring_log_level, dry_run): # pragma: no cover ''' - No destruction is necessary for this monitor. + Remove the monitor handler that was added to the root logger. This prevents the handler from + getting reused by other instances of this monitor. ''' - pass + borgmatic.hooks.monitoring.logs.remove_handler(HANDLER_IDENTIFIER) diff --git a/docs/how-to/monitor-your-backups.md b/docs/how-to/monitor-your-backups.md index 1f0c959e..9bac14d6 100644 --- a/docs/how-to/monitor-your-backups.md +++ b/docs/how-to/monitor-your-backups.md @@ -292,6 +292,27 @@ If you have any issues with the integration, [please contact us](https://torsion.org/borgmatic/#support-and-contributing). +### Sending logs + +New in version 1.9.14 borgmatic +logs are included in the payload data sent to PagerDuty. This means that +(truncated) borgmatic logs, including error messages, show up in the PagerDuty +incident UI and corresponding notification emails. + +You can customize the verbosity of the logs that are sent with borgmatic's +`--monitoring-verbosity` flag. The `--list` and `--stats` flags may also be of +use. See `borgmatic create --help` for more information. + +If you don't want any logs sent, you can disable this feature by setting +`send_logs` to `false`: + +```yaml +pagerduty: + integration_key: a177cad45bd374409f78906a810a3074 + send_logs: false +``` + + ## Pushover hook New in version 1.9.2 diff --git a/tests/unit/hooks/monitoring/test_pagerduty.py b/tests/unit/hooks/monitoring/test_pagerduty.py index f7c73dde..34c55d95 100644 --- a/tests/unit/hooks/monitoring/test_pagerduty.py +++ b/tests/unit/hooks/monitoring/test_pagerduty.py @@ -3,6 +3,46 @@ from flexmock import flexmock from borgmatic.hooks.monitoring import pagerduty as module +def mock_logger(): + logger = flexmock() + logger.should_receive('addHandler') + logger.should_receive('removeHandler') + flexmock(module.logging).should_receive('getLogger').and_return(logger) + + +def test_initialize_monitor_creates_log_handler(): + monitoring_log_level = 1 + + mock_logger() + flexmock(module.borgmatic.hooks.monitoring.logs).should_receive( + 'Forgetful_buffering_handler' + ).once() + + module.initialize_monitor({}, {}, 'test.yaml', monitoring_log_level, dry_run=False) + + +def test_initialize_monitor_creates_log_handler_when_send_logs_true(): + mock_logger() + flexmock(module.borgmatic.hooks.monitoring.logs).should_receive( + 'Forgetful_buffering_handler' + ).once() + + module.initialize_monitor( + {'send_logs': True}, {}, 'test.yaml', monitoring_log_level=1, dry_run=False + ) + + +def test_initialize_monitor_bails_when_send_logs_false(): + mock_logger() + flexmock(module.borgmatic.hooks.monitoring.logs).should_receive( + 'Forgetful_buffering_handler' + ).never() + + module.initialize_monitor( + {'send_logs': False}, {}, 'test.yaml', monitoring_log_level=1, dry_run=False + ) + + def test_ping_monitor_ignores_start_state(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' @@ -39,6 +79,9 @@ def test_ping_monitor_calls_api_for_fail_state(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.monitoring.logs).should_receive( + 'format_buffered_logs_for_payload' + ).and_return('loggy\nlogs') flexmock(module.requests).should_receive('post').and_return(flexmock(ok=True)) module.ping_monitor( @@ -55,6 +98,9 @@ def test_ping_monitor_dry_run_does_not_call_api(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.monitoring.logs).should_receive( + 'format_buffered_logs_for_payload' + ).and_return('loggy\nlogs') flexmock(module.requests).should_receive('post').never() module.ping_monitor( @@ -71,6 +117,9 @@ def test_ping_monitor_with_connection_error_logs_warning(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.monitoring.logs).should_receive( + 'format_buffered_logs_for_payload' + ).and_return('loggy\nlogs') flexmock(module.requests).should_receive('post').and_raise( module.requests.exceptions.ConnectionError ) @@ -108,6 +157,9 @@ def test_ping_monitor_with_other_error_logs_warning(): flexmock(module.borgmatic.hooks.credential.parse).should_receive( 'resolve_credential' ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.monitoring.logs).should_receive( + 'format_buffered_logs_for_payload' + ).and_return('loggy\nlogs') response.should_receive('raise_for_status').and_raise( module.requests.exceptions.RequestException ) From 18ffd96d623caa12763962e129b595492f8ed380 Mon Sep 17 00:00:00 2001 From: Geoff Holden Date: Wed, 5 Mar 2025 22:51:41 -0330 Subject: [PATCH 078/226] Add single quotes around the password. When the DB password uses some special characters, the defaults-extra-file can be incorrect. In the case of a password with the # symbol, anything after that is considered a comment. The single quotes around the password rectify this. --- borgmatic/hooks/data_source/mariadb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index cb05f5b9..eaf05f2d 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -68,7 +68,7 @@ def make_defaults_file_options(username=None, password=None, defaults_extra_file values = '\n'.join( ( (f'user={username}' if username is not None else ''), - (f'password={password}' if password is not None else ''), + (f'password=\'{password}\'' if password is not None else ''), ) ).strip() From a926b413bca6899d8e9f5defa67dfe30dccaeda2 Mon Sep 17 00:00:00 2001 From: Geoff Holden Date: Thu, 6 Mar 2025 09:00:33 -0330 Subject: [PATCH 079/226] Updating automated test, and fixing linting errors. --- borgmatic/hooks/data_source/mariadb.py | 2 +- tests/unit/hooks/data_source/test_mariadb.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index eaf05f2d..8d54e12c 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -68,7 +68,7 @@ def make_defaults_file_options(username=None, password=None, defaults_extra_file values = '\n'.join( ( (f'user={username}' if username is not None else ''), - (f'password=\'{password}\'' if password is not None else ''), + (f"password='{password}'" if password is not None else ''), ) ).strip() diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py index c6b88099..4328f7e6 100644 --- a/tests/unit/hooks/data_source/test_mariadb.py +++ b/tests/unit/hooks/data_source/test_mariadb.py @@ -36,7 +36,7 @@ def test_make_defaults_file_option_with_username_and_password_writes_them_to_fil flexmock(module.os).should_receive('pipe').and_return(read_descriptor, write_descriptor) flexmock(module.os).should_receive('write').with_args( - write_descriptor, b'[client]\nuser=root\npassword=trustsome1' + write_descriptor, b"[client]\nuser=root\npassword='trustsome1'" ).once() flexmock(module.os).should_receive('close') flexmock(module.os).should_receive('set_inheritable') @@ -68,7 +68,7 @@ def test_make_defaults_file_pipe_with_only_password_writes_it_to_file_descriptor flexmock(module.os).should_receive('pipe').and_return(read_descriptor, write_descriptor) flexmock(module.os).should_receive('write').with_args( - write_descriptor, b'[client]\npassword=trustsome1' + write_descriptor, b"[client]\npassword='trustsome1'" ).once() flexmock(module.os).should_receive('close') flexmock(module.os).should_receive('set_inheritable') @@ -84,7 +84,7 @@ def test_make_defaults_file_option_with_defaults_extra_filename_includes_it_in_f flexmock(module.os).should_receive('pipe').and_return(read_descriptor, write_descriptor) flexmock(module.os).should_receive('write').with_args( - write_descriptor, b'!include extra.cnf\n[client]\nuser=root\npassword=trustsome1' + write_descriptor, b"!include extra.cnf\n[client]\nuser=root\npassword='trustsome1'" ).once() flexmock(module.os).should_receive('close') flexmock(module.os).should_receive('set_inheritable') From 1d486d024bed0770c7385d14b3afb6643acb4114 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 6 Mar 2025 10:32:38 -0800 Subject: [PATCH 080/226] Fix a regression in which some MariaDB/MySQL passwords were not escaped correctly (#1017). --- NEWS | 1 + borgmatic/hooks/data_source/mariadb.py | 4 +++- tests/unit/hooks/data_source/test_mariadb.py | 22 +++++++++++++++++--- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index 5a1bd387..dc62723c 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,7 @@ incident UI. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook * #936: Clarify Zabbix monitoring hook documentation about creating items. + * #1017: Fix a regression in which some MariaDB/MySQL passwords were not escaped correctly. 1.9.13 * #975: Add a "compression" option to the PostgreSQL database hook. diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index 8d54e12c..1f140793 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -65,10 +65,12 @@ def make_defaults_file_options(username=None, password=None, defaults_extra_file Do not use the returned value for multiple different command invocations. That will not work because each pipe is "used up" once read. ''' + escaped_password = None if password is None else password.replace('\\', '\\\\') + values = '\n'.join( ( (f'user={username}' if username is not None else ''), - (f"password='{password}'" if password is not None else ''), + (f'password="{escaped_password}"' if escaped_password is not None else ''), ) ).strip() diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py index 4328f7e6..b3051783 100644 --- a/tests/unit/hooks/data_source/test_mariadb.py +++ b/tests/unit/hooks/data_source/test_mariadb.py @@ -36,7 +36,7 @@ def test_make_defaults_file_option_with_username_and_password_writes_them_to_fil flexmock(module.os).should_receive('pipe').and_return(read_descriptor, write_descriptor) flexmock(module.os).should_receive('write').with_args( - write_descriptor, b"[client]\nuser=root\npassword='trustsome1'" + write_descriptor, b'[client]\nuser=root\npassword="trustsome1"' ).once() flexmock(module.os).should_receive('close') flexmock(module.os).should_receive('set_inheritable') @@ -46,6 +46,22 @@ def test_make_defaults_file_option_with_username_and_password_writes_them_to_fil ) +def test_make_defaults_file_option_escapes_password_containing_backslash(): + read_descriptor = 99 + write_descriptor = flexmock() + + flexmock(module.os).should_receive('pipe').and_return(read_descriptor, write_descriptor) + flexmock(module.os).should_receive('write').with_args( + write_descriptor, b'[client]\nuser=root\n' + br'password="trust\\nsome1"' + ).once() + flexmock(module.os).should_receive('close') + flexmock(module.os).should_receive('set_inheritable') + + assert module.make_defaults_file_options(username='root', password=r'trust\nsome1') == ( + '--defaults-extra-file=/dev/fd/99', + ) + + def test_make_defaults_file_pipe_with_only_username_writes_it_to_file_descriptor(): read_descriptor = 99 write_descriptor = flexmock() @@ -68,7 +84,7 @@ def test_make_defaults_file_pipe_with_only_password_writes_it_to_file_descriptor flexmock(module.os).should_receive('pipe').and_return(read_descriptor, write_descriptor) flexmock(module.os).should_receive('write').with_args( - write_descriptor, b"[client]\npassword='trustsome1'" + write_descriptor, b'[client]\npassword="trustsome1"' ).once() flexmock(module.os).should_receive('close') flexmock(module.os).should_receive('set_inheritable') @@ -84,7 +100,7 @@ def test_make_defaults_file_option_with_defaults_extra_filename_includes_it_in_f flexmock(module.os).should_receive('pipe').and_return(read_descriptor, write_descriptor) flexmock(module.os).should_receive('write').with_args( - write_descriptor, b"!include extra.cnf\n[client]\nuser=root\npassword='trustsome1'" + write_descriptor, b'!include extra.cnf\n[client]\nuser=root\npassword="trustsome1"' ).once() flexmock(module.os).should_receive('close') flexmock(module.os).should_receive('set_inheritable') From eb5dc128bf847e09de79cd0994ed527703ba9eda Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 6 Mar 2025 10:34:28 -0800 Subject: [PATCH 081/226] Fix incorrect test name (#1017). --- tests/unit/hooks/data_source/test_mariadb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py index b3051783..871c1969 100644 --- a/tests/unit/hooks/data_source/test_mariadb.py +++ b/tests/unit/hooks/data_source/test_mariadb.py @@ -46,7 +46,7 @@ def test_make_defaults_file_option_with_username_and_password_writes_them_to_fil ) -def test_make_defaults_file_option_escapes_password_containing_backslash(): +def test_make_defaults_file_escapes_password_containing_backslash(): read_descriptor = 99 write_descriptor = flexmock() From c76a108422c31e5f631b215392bc1c5b70caa3e3 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 6 Mar 2025 10:37:00 -0800 Subject: [PATCH 082/226] Link to Zabbix documentation from NEWS. --- NEWS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index dc62723c..a41ebd1e 100644 --- a/NEWS +++ b/NEWS @@ -2,7 +2,8 @@ * #409: With the PagerDuty monitoring hook, send borgmatic logs to PagerDuty so they show up in the incident UI. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook - * #936: Clarify Zabbix monitoring hook documentation about creating items. + * #936: Clarify Zabbix monitoring hook documentation about creating items: + https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#zabbix-hook * #1017: Fix a regression in which some MariaDB/MySQL passwords were not escaped correctly. 1.9.13 From d4f48a3a9e1e42f9de41cfbcc399f9df4c5fb730 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 6 Mar 2025 11:23:24 -0800 Subject: [PATCH 083/226] Initial work on unified command hooks (#790). --- NEWS | 3 + borgmatic/actions/check.py | 19 -- borgmatic/actions/compact.py | 18 -- borgmatic/actions/create.py | 19 -- borgmatic/actions/extract.py | 17 -- borgmatic/actions/prune.py | 17 -- borgmatic/commands/borgmatic.py | 84 ++++--- borgmatic/config/generate.py | 66 +++--- borgmatic/config/normalize.py | 84 +++++++ borgmatic/config/schema.yaml | 254 +++++++++++++++++++--- borgmatic/hooks/command.py | 98 ++++++--- borgmatic/hooks/dispatch.py | 16 +- tests/integration/config/test_generate.py | 32 --- 13 files changed, 469 insertions(+), 258 deletions(-) diff --git a/NEWS b/NEWS index a41ebd1e..b8b9a7cb 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,9 @@ * #409: With the PagerDuty monitoring hook, send borgmatic logs to PagerDuty so they show up in the incident UI. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook + * #790: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more flexible + "commands:". See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ * #936: Clarify Zabbix monitoring hook documentation about creating items: https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#zabbix-hook * #1017: Fix a regression in which some MariaDB/MySQL passwords were not escaped correctly. diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index 2fcca67d..b562d760 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -682,7 +682,6 @@ def run_check( config_filename, repository, config, - hook_context, local_borg_version, check_arguments, global_arguments, @@ -699,15 +698,6 @@ def run_check( ): return - borgmatic.hooks.command.execute_hook( - config.get('before_check'), - config.get('umask'), - config_filename, - 'pre-check', - global_arguments.dry_run, - **hook_context, - ) - logger.info('Running consistency checks') repository_id = borgmatic.borg.check.get_repository_id( @@ -772,12 +762,3 @@ def run_check( borgmatic_runtime_directory, ) write_check_time(make_check_time_path(config, repository_id, 'spot')) - - borgmatic.hooks.command.execute_hook( - config.get('after_check'), - config.get('umask'), - config_filename, - 'post-check', - global_arguments.dry_run, - **hook_context, - ) diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index b0b070bf..a8ab6a6f 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -12,7 +12,6 @@ def run_compact( config_filename, repository, config, - hook_context, local_borg_version, compact_arguments, global_arguments, @@ -28,14 +27,6 @@ def run_compact( ): return - borgmatic.hooks.command.execute_hook( - config.get('before_compact'), - config.get('umask'), - config_filename, - 'pre-compact', - global_arguments.dry_run, - **hook_context, - ) if borgmatic.borg.feature.available(borgmatic.borg.feature.Feature.COMPACT, local_borg_version): logger.info(f'Compacting segments{dry_run_label}') borgmatic.borg.compact.compact_segments( @@ -52,12 +43,3 @@ def run_compact( ) else: # pragma: nocover logger.info('Skipping compact (only available/needed in Borg 1.2+)') - - borgmatic.hooks.command.execute_hook( - config.get('after_compact'), - config.get('umask'), - config_filename, - 'post-compact', - global_arguments.dry_run, - **hook_context, - ) diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index dede448a..11928f62 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -261,7 +261,6 @@ def run_create( repository, config, config_paths, - hook_context, local_borg_version, create_arguments, global_arguments, @@ -279,15 +278,6 @@ def run_create( ): return - borgmatic.hooks.command.execute_hook( - config.get('before_backup'), - config.get('umask'), - config_filename, - 'pre-backup', - global_arguments.dry_run, - **hook_context, - ) - logger.info(f'Creating archive{dry_run_label}') working_directory = borgmatic.config.paths.get_working_directory(config) @@ -343,12 +333,3 @@ def run_create( borgmatic_runtime_directory, global_arguments.dry_run, ) - - borgmatic.hooks.command.execute_hook( - config.get('after_backup'), - config.get('umask'), - config_filename, - 'post-backup', - global_arguments.dry_run, - **hook_context, - ) diff --git a/borgmatic/actions/extract.py b/borgmatic/actions/extract.py index 0216f820..6e2e7900 100644 --- a/borgmatic/actions/extract.py +++ b/borgmatic/actions/extract.py @@ -12,7 +12,6 @@ def run_extract( config_filename, repository, config, - hook_context, local_borg_version, extract_arguments, global_arguments, @@ -22,14 +21,6 @@ def run_extract( ''' Run the "extract" action for the given repository. ''' - borgmatic.hooks.command.execute_hook( - config.get('before_extract'), - config.get('umask'), - config_filename, - 'pre-extract', - global_arguments.dry_run, - **hook_context, - ) if extract_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, extract_arguments.repository ): @@ -56,11 +47,3 @@ def run_extract( strip_components=extract_arguments.strip_components, progress=extract_arguments.progress, ) - borgmatic.hooks.command.execute_hook( - config.get('after_extract'), - config.get('umask'), - config_filename, - 'post-extract', - global_arguments.dry_run, - **hook_context, - ) diff --git a/borgmatic/actions/prune.py b/borgmatic/actions/prune.py index 83dfe39f..e7caa158 100644 --- a/borgmatic/actions/prune.py +++ b/borgmatic/actions/prune.py @@ -11,7 +11,6 @@ def run_prune( config_filename, repository, config, - hook_context, local_borg_version, prune_arguments, global_arguments, @@ -27,14 +26,6 @@ def run_prune( ): return - borgmatic.hooks.command.execute_hook( - config.get('before_prune'), - config.get('umask'), - config_filename, - 'pre-prune', - global_arguments.dry_run, - **hook_context, - ) logger.info(f'Pruning archives{dry_run_label}') borgmatic.borg.prune.prune_archives( global_arguments.dry_run, @@ -46,11 +37,3 @@ def run_prune( local_path=local_path, remote_path=remote_path, ) - borgmatic.hooks.command.execute_hook( - config.get('after_prune'), - config.get('umask'), - config_filename, - 'post-prune', - global_arguments.dry_run, - **hook_context, - ) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index a55f18ab..276f92e0 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -96,6 +96,13 @@ def run_configuration(config_filename, config, config_paths, arguments): f"Skipping {'/'.join(skip_actions)} action{'s' if len(skip_actions) > 1 else ''} due to configured skip_actions" ) + command.execute_hooks( + command.filter_hooks(config.get('commands'), before='configuration', action_names=arguments.keys()), + config.get('umask'), + global_arguments.dry_run, + configuration_filename=config_filename, + ) + try: local_borg_version = borg_version.local_borg_version(config, local_path) logger.debug(f'Borg {local_borg_version}') @@ -226,18 +233,26 @@ def run_configuration(config_filename, config, config_paths, arguments): encountered_error = error yield from log_error_records(f'{config_filename}: Error pinging monitor', error) + if encountered_error: + command.execute_hooks( + command.filter_hooks(config.get('commands'), after='error', action_names=arguments.keys()), + config.get('umask'), + global_arguments.dry_run, + configuration_filename=config_filename, + repository=error_repository, + error=encountered_error, + output=getattr(encountered_error, 'output', ''), + ) + + command.execute_hooks( + command.filter_hooks(config.get('commands'), after='configuration', action_names=arguments.keys()), + config.get('umask'), + global_arguments.dry_run, + configuration_filename=config_filename, + ) + if encountered_error and using_primary_action: try: - command.execute_hook( - config.get('on_error'), - config.get('umask'), - config_filename, - 'on-error', - global_arguments.dry_run, - repository=error_repository, - error=encountered_error, - output=getattr(encountered_error, 'output', ''), - ) dispatch.call_hooks( 'ping_monitor', config, @@ -289,6 +304,7 @@ def run_actions( global_arguments = arguments['global'] dry_run_label = ' (dry run; not making any changes)' if global_arguments.dry_run else '' hook_context = { + 'configuration_filename': config_filename, 'repository_label': repository.get('label', ''), 'log_file': global_arguments.log_file if global_arguments.log_file else '', # Deprecated: For backwards compatibility with borgmatic < 1.6.0. @@ -297,16 +313,24 @@ def run_actions( } skip_actions = set(get_skip_actions(config, arguments)) - command.execute_hook( - config.get('before_actions'), + command.execute_hooks( + command.filter_hooks(config.get('commands'), before='repository', action_names=arguments.keys()), config.get('umask'), - config_filename, - 'pre-actions', global_arguments.dry_run, **hook_context, ) for action_name, action_arguments in arguments.items(): + if action_name == 'global': + continue + + command.execute_hooks( + command.filter_hooks(config.get('commands'), before='action', action_names=arguments.keys()), + config.get('umask'), + global_arguments.dry_run, + **hook_context, + ) + if action_name == 'repo-create' and action_name not in skip_actions: borgmatic.actions.repo_create.run_repo_create( repository, @@ -333,7 +357,6 @@ def run_actions( repository, config, config_paths, - hook_context, local_borg_version, action_arguments, global_arguments, @@ -346,7 +369,6 @@ def run_actions( config_filename, repository, config, - hook_context, local_borg_version, action_arguments, global_arguments, @@ -359,7 +381,6 @@ def run_actions( config_filename, repository, config, - hook_context, local_borg_version, action_arguments, global_arguments, @@ -373,7 +394,6 @@ def run_actions( config_filename, repository, config, - hook_context, local_borg_version, action_arguments, global_arguments, @@ -385,7 +405,6 @@ def run_actions( config_filename, repository, config, - hook_context, local_borg_version, action_arguments, global_arguments, @@ -523,11 +542,16 @@ def run_actions( remote_path, ) - command.execute_hook( - config.get('after_actions'), + command.execute_hooks( + command.filter_hooks(config.get('commands'), after='action', action_names=arguments.keys()), + config.get('umask'), + global_arguments.dry_run, + **hook_context, + ) + + command.execute_hooks( + command.filter_hooks(config.get('commands'), after='repository', action_names=arguments.keys()), config.get('umask'), - config_filename, - 'post-actions', global_arguments.dry_run, **hook_context, ) @@ -813,12 +837,11 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): if 'create' in arguments: try: for config_filename, config in configs.items(): - command.execute_hook( - config.get('before_everything'), + command.execute_hooks( + command.filter_hooks(config.get('commands'), before='everything', action_names=arguments.keys()), config.get('umask'), - config_filename, - 'pre-everything', arguments['global'].dry_run, + configuration_filename=config_filename, ) except (CalledProcessError, ValueError, OSError) as error: yield from log_error_records('Error running pre-everything hook', error) @@ -865,12 +888,11 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): if 'create' in arguments: try: for config_filename, config in configs.items(): - command.execute_hook( - config.get('after_everything'), + command.execute_hooks( + command.filter_hooks(config.get('commands'), after='everything', action_names=arguments.keys()), config.get('umask'), - config_filename, - 'post-everything', arguments['global'].dry_run, + configuration_filename=config_filename, ) except (CalledProcessError, ValueError, OSError) as error: yield from log_error_records('Error running post-everything hook', error) diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index 6b375b76..11cf0964 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -1,5 +1,6 @@ import collections import io +import itertools import os import re @@ -24,20 +25,28 @@ def insert_newline_before_comment(config, field_name): def get_properties(schema): ''' Given a schema dict, return its properties. But if it's got sub-schemas with multiple different - potential properties, returned their merged properties instead. + potential properties, returned their merged properties instead (interleaved so the first + properties of each sub-schema come first). The idea is that the user should see all possible + options even if they're not all possible together. ''' if 'oneOf' in schema: return dict( - collections.ChainMap(*[sub_schema['properties'] for sub_schema in schema['oneOf']]) + item + for item in itertools.chain( + *itertools.zip_longest( + *[sub_schema['properties'].items() for sub_schema in schema['oneOf']] + ) + ) + if item is not None ) return schema['properties'] -def schema_to_sample_configuration(schema, level=0, parent_is_sequence=False): +def schema_to_sample_configuration(schema, source_config, level=0, parent_is_sequence=False): ''' - Given a loaded configuration schema, generate and return sample config for it. Include comments - for each option based on the schema "description". + Given a loaded configuration schema and a source configuration, generate and return sample + config for the schema. Include comments for each option based on the schema "description". ''' schema_type = schema.get('type') example = schema.get('example') @@ -47,19 +56,22 @@ def schema_to_sample_configuration(schema, level=0, parent_is_sequence=False): if schema_type == 'array' or (isinstance(schema_type, list) and 'array' in schema_type): config = ruamel.yaml.comments.CommentedSeq( - [schema_to_sample_configuration(schema['items'], level, parent_is_sequence=True)] + [schema_to_sample_configuration(schema['items'], source_config, level, parent_is_sequence=True)] ) add_comments_to_configuration_sequence(config, schema, indent=(level * INDENT)) elif schema_type == 'object' or (isinstance(schema_type, list) and 'object' in schema_type): + if source_config and isinstance(source_config, list) and isinstance(source_config[0], dict): + source_config = dict(collections.ChainMap(*source_config)) + config = ruamel.yaml.comments.CommentedMap( [ - (field_name, schema_to_sample_configuration(sub_schema, level + 1)) + (field_name, schema_to_sample_configuration(sub_schema, source_config.get(field_name, {}), level + 1)) for field_name, sub_schema in get_properties(schema).items() ] ) indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0) add_comments_to_configuration_object( - config, schema, indent=indent, skip_first=parent_is_sequence + config, schema, source_config, indent=indent, skip_first=parent_is_sequence ) else: raise ValueError(f'Schema at level {level} is unsupported: {schema}') @@ -179,14 +191,19 @@ def add_comments_to_configuration_sequence(config, schema, indent=0): return -REQUIRED_KEYS = {'source_directories', 'repositories', 'keep_daily'} +DEFAULT_KEYS = {'source_directories', 'repositories', 'keep_daily'} COMMENTED_OUT_SENTINEL = 'COMMENT_OUT' -def add_comments_to_configuration_object(config, schema, indent=0, skip_first=False): +def add_comments_to_configuration_object(config, schema, source_config, indent=0, skip_first=False): ''' Using descriptions from a schema as a source, add those descriptions as comments to the given - config mapping, before each field. Indent the comment the given number of characters. + configuration dict, putting them before each field. Indent the comment the given number of + characters. + + And a sentinel for commenting out options that are neither in DEFAULT_KEYS nor the the given + source configuration dict. The idea is that any options used in the source configuration should + stay active in the generated configuration. ''' for index, field_name in enumerate(config.keys()): if skip_first and index == 0: @@ -195,10 +212,10 @@ def add_comments_to_configuration_object(config, schema, indent=0, skip_first=Fa field_schema = get_properties(schema).get(field_name, {}) description = field_schema.get('description', '').strip() - # If this is an optional key, add an indicator to the comment flagging it to be commented + # If this isn't a default key, add an indicator to the comment flagging it to be commented # out from the sample configuration. This sentinel is consumed by downstream processing that # does the actual commenting out. - if field_name not in REQUIRED_KEYS: + if field_name not in DEFAULT_KEYS and field_name not in source_config: description = ( '\n'.join((description, COMMENTED_OUT_SENTINEL)) if description @@ -218,21 +235,6 @@ def add_comments_to_configuration_object(config, schema, indent=0, skip_first=Fa RUAMEL_YAML_COMMENTS_INDEX = 1 -def remove_commented_out_sentinel(config, field_name): - ''' - Given a configuration CommentedMap and a top-level field name in it, remove any "commented out" - sentinel found at the end of its YAML comments. This prevents the given field name from getting - commented out by downstream processing that consumes the sentinel. - ''' - try: - last_comment_value = config.ca.items[field_name][RUAMEL_YAML_COMMENTS_INDEX][-1].value - except KeyError: - return - - if last_comment_value == f'# {COMMENTED_OUT_SENTINEL}\n': - config.ca.items[field_name][RUAMEL_YAML_COMMENTS_INDEX].pop() - - def merge_source_configuration_into_destination(destination_config, source_config): ''' Deep merge the given source configuration dict into the destination configuration CommentedMap, @@ -247,12 +249,6 @@ def merge_source_configuration_into_destination(destination_config, source_confi return source_config for field_name, source_value in source_config.items(): - # Since this key/value is from the source configuration, leave it uncommented and remove any - # sentinel that would cause it to get commented out. - remove_commented_out_sentinel( - ruamel.yaml.comments.CommentedMap(destination_config), field_name - ) - # This is a mapping. Recurse for this key/value. if isinstance(source_value, collections.abc.Mapping): destination_config[field_name] = merge_source_configuration_into_destination( @@ -298,7 +294,7 @@ def generate_sample_configuration( normalize.normalize(source_filename, source_config) destination_config = merge_source_configuration_into_destination( - schema_to_sample_configuration(schema), source_config + schema_to_sample_configuration(schema, source_config), source_config ) if dry_run: diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index a869b24a..1f34c071 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -58,6 +58,89 @@ def normalize_sections(config_filename, config): return [] +def make_command_hook_deprecation_log(config_filename, option_name): + ''' + Given a configuration filename and the name of a configuration option, return a deprecation + warning log for it. + ''' + return logging.makeLogRecord( + dict( + levelno=logging.WARNING, + levelname='WARNING', + msg=f'{config_filename}: {option_name} is deprecated and support will be removed from a future release. Use commands: instead.', + ) + ) + + +def normalize_commands(config_filename, config): + ''' + Given a configuration filename and a configuration dict, transform any "before_*"- and + "after_*"-style command hooks into "commands:". + ''' + logs = [] + + # Normalize "before_actions" and "after_actions". + for preposition in ('before', 'after'): + option_name = f'{preposition}_actions' + commands = config.pop(option_name, None) + + if commands: + logs.append(make_command_hook_deprecation_log(config_filename, option_name)) + config.setdefault('commands', []).append( + { + preposition: 'repository', + 'run': commands, + } + ) + + # Normalize "before_backup", "before_prune", "after_backup", "after_prune", etc. + for action_name in ('create', 'prune', 'compact', 'check', 'extract'): + for preposition in ('before', 'after'): + option_name = f'{preposition}_{"backup" if action_name == "create" else action_name}' + commands = config.pop(option_name, None) + + if not commands: + continue + + logs.append(make_command_hook_deprecation_log(config_filename, option_name)) + config.setdefault('commands', []).append( + { + preposition: 'action', + 'when': [action_name], + 'run': commands, + } + ) + + # Normalize "on_error". + commands = config.pop('on_error', None) + + if commands: + logs.append(make_command_hook_deprecation_log(config_filename, 'on_error')) + config.setdefault('commands', []).append( + { + 'after': 'error', + 'when': ['create', 'prune', 'compact', 'check'], + 'run': commands, + } + ) + + # Normalize "before_everything" and "after_everything". + for preposition in ('before', 'after'): + option_name = f'{preposition}_everything' + commands = config.pop(option_name, None) + + if commands: + logs.append(make_command_hook_deprecation_log(config_filename, option_name)) + config.setdefault('commands', []).append( + { + preposition: 'everything', + 'run': commands, + } + ) + + return logs + + def normalize(config_filename, config): ''' Given a configuration filename and a configuration dict of its loaded contents, apply particular @@ -67,6 +150,7 @@ def normalize(config_filename, config): Raise ValueError the configuration cannot be normalized. ''' logs = normalize_sections(config_filename, config) + logs += normalize_commands(config_filename, config) if config.get('borgmatic_source_directory'): logs.append( diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index f6ec574e..4d62e5fe 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -796,8 +796,9 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute before all - the actions for each repository. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute before all the actions for each + repository. example: - "echo Starting actions." before_backup: @@ -805,8 +806,9 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute before - creating a backup, run once per repository. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute before creating a backup, run once + per repository. example: - "echo Starting a backup." before_prune: @@ -814,8 +816,9 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute before - pruning, run once per repository. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute before pruning, run once per + repository. example: - "echo Starting pruning." before_compact: @@ -823,8 +826,9 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute before - compaction, run once per repository. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute before compaction, run once per + repository. example: - "echo Starting compaction." before_check: @@ -832,8 +836,9 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute before - consistency checks, run once per repository. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute before consistency checks, run once + per repository. example: - "echo Starting checks." before_extract: @@ -841,8 +846,9 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute before - extracting a backup, run once per repository. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute before extracting a backup, run once + per repository. example: - "echo Starting extracting." after_backup: @@ -850,8 +856,9 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute after - creating a backup, run once per repository. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute after creating a backup, run once per + repository. example: - "echo Finished a backup." after_compact: @@ -859,8 +866,9 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute after - compaction, run once per repository. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute after compaction, run once per + repository. example: - "echo Finished compaction." after_prune: @@ -868,8 +876,9 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute after - pruning, run once per repository. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute after pruning, run once per + repository. example: - "echo Finished pruning." after_check: @@ -877,8 +886,9 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute after - consistency checks, run once per repository. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute after consistency checks, run once + per repository. example: - "echo Finished checks." after_extract: @@ -886,8 +896,9 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute after - extracting a backup, run once per repository. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute after extracting a backup, run once + per repository. example: - "echo Finished extracting." after_actions: @@ -895,8 +906,9 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute after all - actions for each repository. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute after all actions for each + repository. example: - "echo Finished actions." on_error: @@ -904,9 +916,10 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute when an - exception occurs during a "create", "prune", "compact", or "check" - action or an associated before/after hook. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute when an exception occurs during a + "create", "prune", "compact", or "check" action or an associated + before/after hook. example: - "echo Error during create/prune/compact/check." before_everything: @@ -914,10 +927,10 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute before - running all actions (if one of them is "create"). These are - collected from all configuration files and then run once before all - of them (prior to all actions). + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute before running all actions (if one of + them is "create"). These are collected from all configuration files + and then run once before all of them (prior to all actions). example: - "echo Starting actions." after_everything: @@ -925,12 +938,183 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute after - running all actions (if one of them is "create"). These are - collected from all configuration files and then run once after all - of them (after any action). + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute after running all actions (if one of + them is "create"). These are collected from all configuration files + and then run once after all of them (after any action). example: - "echo Completed actions." + commands: + type: array + items: + type: object + oneOf: + - required: [before, run] + additionalProperties: false + properties: + before: + type: string + enum: + - action + - repository + - configuration + - everything + - dump_data_sources + - restore_data_source_dump + description: | + Name for the point in borgmatic's execution before + which the commands should be run (required if + "after" isn't set): + * "action" runs before each action for each + repository. + * "repository" runs before all actions for each + repository. + * "configuration" runs before all actions and + repositories in the current configuration file. + * "everything" runs before all configuration + files. + * "dump_data_sources" runs before each data + source is dumped. + * "restore_data_source_dumps" runs before each + data source is restored. + example: action + hooks: + type: array + items: + type: string + description: | + List of names of other hooks that this command + hook applies to. Defaults to all hooks of the + relevant type. Only supported for + "dump_data_sources" and + "restore_data_source_dumps" hooks. + example: postgresql + when: + type: array + items: + type: string + enum: + - repo-create + - transfer + - prune + - compact + - create + - check + - delete + - extract + - config + - export-tar + - mount + - umount + - repo-delete + - restore + - repo-list + - list + - repo-info + - info + - break-lock + - key + - borg + description: | + List of actions for which the commands will be run. Defaults to + running for all actions. Ignored for "dump_data_sources" and + "restore_data_source_dump", which by their nature only run for + particular actions. + example: [create, prune, compact, check] + run: + type: array + items: + type: string + description: | + List of one or more shell commands or scripts to + run when this command hook is triggered. Required. + example: + - "echo Doing stuff." + - required: [after, run] + additionalProperties: false + properties: + after: + type: string + enum: + - action + - repository + - configuration + - everything + - error + - dump_data_sources + - restore_data_source_dump + description: | + Name for the point in borgmatic's execution after + which the commands should be run (required if + "before" isn't set): + * "action" runs after each action for each + repository. + * "repository" runs after all actions for each + repository. + * "configuration" runs after all actions and + repositories in the current configuration file. + * "everything" runs after all configuration + files. + * "error" runs after an error occurs. + * "dump_data_sources" runs after each data + source is dumped. + * "restore_data_source_dumps" runs after each + data source is restored. + example: action + hooks: + type: array + items: + type: string + description: | + List of names of other hooks that this command + hook applies to. Defaults to all hooks of the + relevant type. Only supported for + "dump_data_sources" and + "restore_data_source_dumps" hooks. + example: postgresql + when: + type: array + items: + type: string + enum: + - repo-create + - transfer + - prune + - compact + - create + - check + - delete + - extract + - config + - export-tar + - mount + - umount + - repo-delete + - restore + - repo-list + - list + - repo-info + - info + - break-lock + - key + - borg + description: | + List of actions for which the commands will be run. + Defaults to running for all actions. + example: [create, prune, compact, check] + run: + type: array + items: + type: string + description: | + List of one or more shell commands or scripts to + run when this command hook is triggered. Required. + example: + - "echo Doing stuff." + description: | + List of one or more command hooks to execute, triggered at + particular points during borgmatic's execution. For each command + hook, specify one of "before" or "after", not both. bootstrap: type: object properties: diff --git a/borgmatic/hooks/command.py b/borgmatic/hooks/command.py index 9e605290..a423e0c3 100644 --- a/borgmatic/hooks/command.py +++ b/borgmatic/hooks/command.py @@ -5,6 +5,7 @@ import shlex import sys import borgmatic.execute +import borgmatic.logger logger = logging.getLogger(__name__) @@ -44,54 +45,83 @@ def make_environment(current_environment, sys_module=sys): return environment -def execute_hook(commands, umask, config_filename, description, dry_run, **context): +def filter_hooks(command_hooks, before=None, after=None, hook_name=None, action_names=None): ''' - Given a list of hook commands to execute, a umask to execute with (or None), a config filename, - a hook description, and whether this is a dry run, run the given commands. Or, don't run them - if this is a dry run. + Given a sequence of command hook dicts from configuration and one or more filters (before name, + after name, calling hook name, or a sequence of action names), filter down the command hooks to + just the ones that match the given filters. + ''' + return tuple( + hook_config + for hook_config in command_hooks + for config_hook_names in (hook_config.get('hooks'),) + for config_action_names in (hook_config.get('when'),) + if before is None or hook_config.get('before') == before + if after is None or hook_config.get('after') == after + if hook_name is None or config_hook_names is None or hook_name in config_hook_names + if action_names is None or config_action_names is None or set(config_action_names or ()).intersection(set(action_names)) + ) + + +def execute_hooks(command_hooks, umask, dry_run, **context): + ''' + Given a sequence of command hook dicts from configuration, a umask to execute with (or None), + and whether this is a dry run, run the commands for each hook. Or don't run them if this is a + dry run. The context contains optional values interpolated by name into the hook commands. - Raise ValueError if the umask cannot be parsed. + Raise ValueError if the umask cannot be parsed or a hook is invalid. Raise subprocesses.CalledProcessError if an error occurs in a hook. ''' - if not commands: - logger.debug(f'No commands to run for {description} hook') - return + borgmatic.logger.add_custom_log_levels() dry_run_label = ' (dry run; not actually running hooks)' if dry_run else '' - context['configuration_filename'] = config_filename - commands = [interpolate_context(description, command, context) for command in commands] + for hook_config in command_hooks: + commands = hook_config.get('run') - if len(commands) == 1: - logger.info(f'Running command for {description} hook{dry_run_label}') - else: - logger.info( - f'Running {len(commands)} commands for {description} hook{dry_run_label}', - ) + if 'before' in hook_config: + description = f'before {hook_config.get("before")}' + elif 'after' in hook_config: + description = f'after {hook_config.get("after")}' + else: + raise ValueError('Invalid hook configuration: {hook_config}') - if umask: - parsed_umask = int(str(umask), 8) - logger.debug(f'Set hook umask to {oct(parsed_umask)}') - original_umask = os.umask(parsed_umask) - else: - original_umask = None + if not commands: + logger.debug(f'No commands to run for {description} hook') + return - try: - for command in commands: - if dry_run: - continue + commands = [interpolate_context(description, command, context) for command in commands] - borgmatic.execute.execute_command( - [command], - output_log_level=(logging.ERROR if description == 'on-error' else logging.WARNING), - shell=True, - environment=make_environment(os.environ), + if len(commands) == 1: + logger.info(f'Running command for {description} hook{dry_run_label}') + else: + logger.info( + f'Running {len(commands)} commands for {description} hook{dry_run_label}', ) - finally: - if original_umask: - os.umask(original_umask) + + if umask: + parsed_umask = int(str(umask), 8) + logger.debug(f'Setting hook umask to {oct(parsed_umask)}') + original_umask = os.umask(parsed_umask) + else: + original_umask = None + + try: + for command in commands: + if dry_run: + continue + + borgmatic.execute.execute_command( + [command], + output_log_level=(logging.ERROR if hook_config.get('after') == 'error' else logging.ANSWER), + shell=True, + environment=make_environment(os.environ), + ) + finally: + if original_umask: + os.umask(original_umask) def considered_soft_failure(error): diff --git a/borgmatic/hooks/dispatch.py b/borgmatic/hooks/dispatch.py index 26b0832c..970d5d5d 100644 --- a/borgmatic/hooks/dispatch.py +++ b/borgmatic/hooks/dispatch.py @@ -3,6 +3,7 @@ import importlib import logging import pkgutil +import borgmatic.hooks.command import borgmatic.hooks.credential import borgmatic.hooks.data_source import borgmatic.hooks.monitoring @@ -62,7 +63,20 @@ def call_hook(function_name, config, hook_name, *args, **kwargs): logger.debug(f'Calling {hook_name} hook function {function_name}') - return getattr(module, function_name)(hook_config, config, *args, **kwargs) + borgmatic.hooks.command.execute_hooks( + borgmatic.hooks.command.filter_hooks(config.get('commands'), before=function_name, hook_name=hook_name), + config.get('umask'), + dry_run=False, # FIXME: Need to get this from somewhere. + ) + + try: + return getattr(module, function_name)(hook_config, config, *args, **kwargs) + finally: + borgmatic.hooks.command.execute_hooks( + borgmatic.hooks.command.filter_hooks(config.get('commands'), after=function_name, hook_name=hook_name), + config.get('umask'), + dry_run=False, # FIXME: Need to get this from somewhere. + ) def call_hooks(function_name, config, hook_type, *args, **kwargs): diff --git a/tests/integration/config/test_generate.py b/tests/integration/config/test_generate.py index 0781e8d1..2db4823a 100644 --- a/tests/integration/config/test_generate.py +++ b/tests/integration/config/test_generate.py @@ -170,38 +170,6 @@ def test_add_comments_to_configuration_object_with_skip_first_does_not_raise(): module.add_comments_to_configuration_object(config, schema, skip_first=True) -def test_remove_commented_out_sentinel_keeps_other_comments(): - field_name = 'foo' - config = module.ruamel.yaml.comments.CommentedMap([(field_name, 33)]) - config.yaml_set_comment_before_after_key(key=field_name, before='Actual comment.\nCOMMENT_OUT') - - module.remove_commented_out_sentinel(config, field_name) - - comments = config.ca.items[field_name][module.RUAMEL_YAML_COMMENTS_INDEX] - assert len(comments) == 1 - assert comments[0].value == '# Actual comment.\n' - - -def test_remove_commented_out_sentinel_without_sentinel_keeps_other_comments(): - field_name = 'foo' - config = module.ruamel.yaml.comments.CommentedMap([(field_name, 33)]) - config.yaml_set_comment_before_after_key(key=field_name, before='Actual comment.') - - module.remove_commented_out_sentinel(config, field_name) - - comments = config.ca.items[field_name][module.RUAMEL_YAML_COMMENTS_INDEX] - assert len(comments) == 1 - assert comments[0].value == '# Actual comment.\n' - - -def test_remove_commented_out_sentinel_on_unknown_field_does_not_raise(): - field_name = 'foo' - config = module.ruamel.yaml.comments.CommentedMap([(field_name, 33)]) - config.yaml_set_comment_before_after_key(key=field_name, before='Actual comment.') - - module.remove_commented_out_sentinel(config, 'unknown') - - def test_generate_sample_configuration_does_not_raise(): builtins = flexmock(sys.modules['builtins']) builtins.should_receive('open').with_args('schema.yaml').and_return('') From 10bd1c7b41476feef6fcc4b54e068e86b8de1aad Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 6 Mar 2025 22:53:19 -0800 Subject: [PATCH 084/226] Remove restore_data_source_dump as a command hook for now (#790). --- borgmatic/config/schema.yaml | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 4d62e5fe..e46656c1 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -960,7 +960,6 @@ properties: - configuration - everything - dump_data_sources - - restore_data_source_dump description: | Name for the point in borgmatic's execution before which the commands should be run (required if @@ -975,8 +974,6 @@ properties: files. * "dump_data_sources" runs before each data source is dumped. - * "restore_data_source_dumps" runs before each - data source is restored. example: action hooks: type: array @@ -985,9 +982,8 @@ properties: description: | List of names of other hooks that this command hook applies to. Defaults to all hooks of the - relevant type. Only supported for - "dump_data_sources" and - "restore_data_source_dumps" hooks. + relevant type. Only supported for the + "dump_data_sources" hook. example: postgresql when: type: array @@ -1016,10 +1012,10 @@ properties: - key - borg description: | - List of actions for which the commands will be run. Defaults to - running for all actions. Ignored for "dump_data_sources" and - "restore_data_source_dump", which by their nature only run for - particular actions. + List of actions for which the commands will be + run. Defaults to running for all actions. Ignored + for "dump_data_sources", which by its nature only + runs for particular actions. example: [create, prune, compact, check] run: type: array @@ -1042,7 +1038,6 @@ properties: - everything - error - dump_data_sources - - restore_data_source_dump description: | Name for the point in borgmatic's execution after which the commands should be run (required if @@ -1058,8 +1053,6 @@ properties: * "error" runs after an error occurs. * "dump_data_sources" runs after each data source is dumped. - * "restore_data_source_dumps" runs after each - data source is restored. example: action hooks: type: array @@ -1068,9 +1061,8 @@ properties: description: | List of names of other hooks that this command hook applies to. Defaults to all hooks of the - relevant type. Only supported for - "dump_data_sources" and - "restore_data_source_dumps" hooks. + relevant type. Only supported for the + "dump_data_sources" hook. example: postgresql when: type: array From e06c6740f20c08b831d7243934bdb8e92ee5eb8e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 7 Mar 2025 10:33:39 -0800 Subject: [PATCH 085/226] Switch to context manager for running "dump_data_sources" before/after hooks (#790). --- borgmatic/commands/borgmatic.py | 36 +++- borgmatic/config/generate.py | 13 +- borgmatic/config/normalize.py | 2 +- borgmatic/hooks/command.py | 70 +++++++- borgmatic/hooks/data_source/bootstrap.py | 68 ++++---- borgmatic/hooks/data_source/btrfs.py | 58 ++++--- borgmatic/hooks/data_source/lvm.py | 132 ++++++++------- borgmatic/hooks/data_source/mariadb.py | 100 +++++------ borgmatic/hooks/data_source/mongodb.py | 77 +++++---- borgmatic/hooks/data_source/mysql.py | 100 +++++------ borgmatic/hooks/data_source/postgresql.py | 192 ++++++++++++---------- borgmatic/hooks/data_source/sqlite.py | 96 ++++++----- borgmatic/hooks/data_source/zfs.py | 110 +++++++------ borgmatic/hooks/dispatch.py | 15 +- 14 files changed, 615 insertions(+), 454 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 276f92e0..5b9527b8 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -97,7 +97,9 @@ def run_configuration(config_filename, config, config_paths, arguments): ) command.execute_hooks( - command.filter_hooks(config.get('commands'), before='configuration', action_names=arguments.keys()), + command.filter_hooks( + config.get('commands'), before='configuration', action_names=arguments.keys() + ), config.get('umask'), global_arguments.dry_run, configuration_filename=config_filename, @@ -235,7 +237,9 @@ def run_configuration(config_filename, config, config_paths, arguments): if encountered_error: command.execute_hooks( - command.filter_hooks(config.get('commands'), after='error', action_names=arguments.keys()), + command.filter_hooks( + config.get('commands'), after='error', action_names=arguments.keys() + ), config.get('umask'), global_arguments.dry_run, configuration_filename=config_filename, @@ -245,7 +249,9 @@ def run_configuration(config_filename, config, config_paths, arguments): ) command.execute_hooks( - command.filter_hooks(config.get('commands'), after='configuration', action_names=arguments.keys()), + command.filter_hooks( + config.get('commands'), after='configuration', action_names=arguments.keys() + ), config.get('umask'), global_arguments.dry_run, configuration_filename=config_filename, @@ -314,7 +320,9 @@ def run_actions( skip_actions = set(get_skip_actions(config, arguments)) command.execute_hooks( - command.filter_hooks(config.get('commands'), before='repository', action_names=arguments.keys()), + command.filter_hooks( + config.get('commands'), before='repository', action_names=arguments.keys() + ), config.get('umask'), global_arguments.dry_run, **hook_context, @@ -325,7 +333,9 @@ def run_actions( continue command.execute_hooks( - command.filter_hooks(config.get('commands'), before='action', action_names=arguments.keys()), + command.filter_hooks( + config.get('commands'), before='action', action_names=arguments.keys() + ), config.get('umask'), global_arguments.dry_run, **hook_context, @@ -543,14 +553,18 @@ def run_actions( ) command.execute_hooks( - command.filter_hooks(config.get('commands'), after='action', action_names=arguments.keys()), + command.filter_hooks( + config.get('commands'), after='action', action_names=arguments.keys() + ), config.get('umask'), global_arguments.dry_run, **hook_context, ) command.execute_hooks( - command.filter_hooks(config.get('commands'), after='repository', action_names=arguments.keys()), + command.filter_hooks( + config.get('commands'), after='repository', action_names=arguments.keys() + ), config.get('umask'), global_arguments.dry_run, **hook_context, @@ -838,7 +852,9 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): try: for config_filename, config in configs.items(): command.execute_hooks( - command.filter_hooks(config.get('commands'), before='everything', action_names=arguments.keys()), + command.filter_hooks( + config.get('commands'), before='everything', action_names=arguments.keys() + ), config.get('umask'), arguments['global'].dry_run, configuration_filename=config_filename, @@ -889,7 +905,9 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): try: for config_filename, config in configs.items(): command.execute_hooks( - command.filter_hooks(config.get('commands'), after='everything', action_names=arguments.keys()), + command.filter_hooks( + config.get('commands'), after='everything', action_names=arguments.keys() + ), config.get('umask'), arguments['global'].dry_run, configuration_filename=config_filename, diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index 11cf0964..d3096717 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -56,7 +56,11 @@ def schema_to_sample_configuration(schema, source_config, level=0, parent_is_seq if schema_type == 'array' or (isinstance(schema_type, list) and 'array' in schema_type): config = ruamel.yaml.comments.CommentedSeq( - [schema_to_sample_configuration(schema['items'], source_config, level, parent_is_sequence=True)] + [ + schema_to_sample_configuration( + schema['items'], source_config, level, parent_is_sequence=True + ) + ] ) add_comments_to_configuration_sequence(config, schema, indent=(level * INDENT)) elif schema_type == 'object' or (isinstance(schema_type, list) and 'object' in schema_type): @@ -65,7 +69,12 @@ def schema_to_sample_configuration(schema, source_config, level=0, parent_is_seq config = ruamel.yaml.comments.CommentedMap( [ - (field_name, schema_to_sample_configuration(sub_schema, source_config.get(field_name, {}), level + 1)) + ( + field_name, + schema_to_sample_configuration( + sub_schema, source_config.get(field_name, {}), level + 1 + ), + ) for field_name, sub_schema in get_properties(schema).items() ] ) diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index 1f34c071..7da0023d 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -98,7 +98,7 @@ def normalize_commands(config_filename, config): for preposition in ('before', 'after'): option_name = f'{preposition}_{"backup" if action_name == "create" else action_name}' commands = config.pop(option_name, None) - + if not commands: continue diff --git a/borgmatic/hooks/command.py b/borgmatic/hooks/command.py index a423e0c3..d46184c1 100644 --- a/borgmatic/hooks/command.py +++ b/borgmatic/hooks/command.py @@ -59,7 +59,9 @@ def filter_hooks(command_hooks, before=None, after=None, hook_name=None, action_ if before is None or hook_config.get('before') == before if after is None or hook_config.get('after') == after if hook_name is None or config_hook_names is None or hook_name in config_hook_names - if action_names is None or config_action_names is None or set(config_action_names or ()).intersection(set(action_names)) + if action_names is None + or config_action_names is None + or set(config_action_names or ()).intersection(set(action_names)) ) @@ -115,7 +117,9 @@ def execute_hooks(command_hooks, umask, dry_run, **context): borgmatic.execute.execute_command( [command], - output_log_level=(logging.ERROR if hook_config.get('after') == 'error' else logging.ANSWER), + output_log_level=( + logging.ERROR if hook_config.get('after') == 'error' else logging.ANSWER + ), shell=True, environment=make_environment(os.environ), ) @@ -124,6 +128,68 @@ def execute_hooks(command_hooks, umask, dry_run, **context): os.umask(original_umask) +class Before_after_hooks: + ''' + A Python context manager for executing command hooks both before and after the wrapped code. + + Example use as a context manager: + + + with borgmatic.hooks.command.Before_after_hooks( + command_hooks=config.get('commands'), + before_after='do_stuff', + hook_name='myhook', + umask=config.get('umask'), + dry_run=dry_run, + ): + do() + some() + stuff() + + With that context manager in place, "before" command hooks execute before the wrapped code runs, + and "after" command hooks execute after the wrapped code completes. + ''' + + def __init__(self, command_hooks, before_after, hook_name, umask, dry_run, **context): + ''' + Given a sequence of command hook configuration dicts, the before/after name, the name of the + calling hook, a umask to run commands with, a dry run flag, and any context for the executed + commands, save those data points for use below. + ''' + self.command_hooks = command_hooks + self.before_after = before_after + self.hook_name = hook_name + self.umask = umask + self.dry_run = dry_run + self.context = context + + def __enter__(self): + ''' + Run the configured "before" command hooks that match the initialized data points. + ''' + execute_hooks( + borgmatic.hooks.command.filter_hooks( + self.command_hooks, before=self.before_after, hook_name=self.hook_name + ), + self.umask, + self.dry_run, + **self.context, + ) + + def __exit__(self, exception, value, traceback): + ''' + Run the configured "after" command hooks that match the initialized data points. + ''' + execute_hooks( + borgmatic.hooks.command.filter_hooks( + self.command_hooks, after=self.before_after, hook_name=self.hook_name + ), + self.umask, + self.dry_run, + **self.context, + ) + + def considered_soft_failure(error): ''' Given a configuration filename and an exception object, return whether the exception object diff --git a/borgmatic/hooks/data_source/bootstrap.py b/borgmatic/hooks/data_source/bootstrap.py index fe779f0e..5a2b4f09 100644 --- a/borgmatic/hooks/data_source/bootstrap.py +++ b/borgmatic/hooks/data_source/bootstrap.py @@ -6,6 +6,7 @@ import os import borgmatic.borg.pattern import borgmatic.config.paths +import borgmatic.hooks.command logger = logging.getLogger(__name__) @@ -37,39 +38,46 @@ def dump_data_sources( if hook_config and hook_config.get('store_config_files') is False: return [] - borgmatic_manifest_path = os.path.join( - borgmatic_runtime_directory, 'bootstrap', 'manifest.json' - ) + with borgmatic.hooks.command.Before_after_hooks( + command_hooks=config.get('commands'), + before_after='dump_data_sources', + hook_name='bootstrap', + umask=config.get('umask'), + dry_run=dry_run, + ): + borgmatic_manifest_path = os.path.join( + borgmatic_runtime_directory, 'bootstrap', 'manifest.json' + ) + + if dry_run: + return [] + + os.makedirs(os.path.dirname(borgmatic_manifest_path), exist_ok=True) + + with open(borgmatic_manifest_path, 'w') as manifest_file: + json.dump( + { + 'borgmatic_version': importlib.metadata.version('borgmatic'), + 'config_paths': config_paths, + }, + manifest_file, + ) + + patterns.extend( + borgmatic.borg.pattern.Pattern( + config_path, source=borgmatic.borg.pattern.Pattern_source.HOOK + ) + for config_path in config_paths + ) + patterns.append( + borgmatic.borg.pattern.Pattern( + os.path.join(borgmatic_runtime_directory, 'bootstrap'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, + ) + ) - if dry_run: return [] - os.makedirs(os.path.dirname(borgmatic_manifest_path), exist_ok=True) - - with open(borgmatic_manifest_path, 'w') as manifest_file: - json.dump( - { - 'borgmatic_version': importlib.metadata.version('borgmatic'), - 'config_paths': config_paths, - }, - manifest_file, - ) - - patterns.extend( - borgmatic.borg.pattern.Pattern( - config_path, source=borgmatic.borg.pattern.Pattern_source.HOOK - ) - for config_path in config_paths - ) - patterns.append( - borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'bootstrap'), - source=borgmatic.borg.pattern.Pattern_source.HOOK, - ) - ) - - return [] - def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, dry_run): ''' diff --git a/borgmatic/hooks/data_source/btrfs.py b/borgmatic/hooks/data_source/btrfs.py index ec6d33df..7023fea6 100644 --- a/borgmatic/hooks/data_source/btrfs.py +++ b/borgmatic/hooks/data_source/btrfs.py @@ -9,6 +9,7 @@ import subprocess import borgmatic.borg.pattern import borgmatic.config.paths import borgmatic.execute +import borgmatic.hooks.command import borgmatic.hooks.data_source.snapshot logger = logging.getLogger(__name__) @@ -204,41 +205,48 @@ def dump_data_sources( If this is a dry run, then don't actually snapshot anything. ''' - dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' - logger.info(f'Snapshotting Btrfs subvolumes{dry_run_label}') + with borgmatic.hooks.command.Before_after_hooks( + command_hooks=config.get('commands'), + before_after='dump_data_sources', + hook_name='btrfs', + umask=config.get('umask'), + dry_run=dry_run, + ): + dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' + logger.info(f'Snapshotting Btrfs subvolumes{dry_run_label}') - # Based on the configured patterns, determine Btrfs subvolumes to backup. Only consider those - # patterns that came from actual user configuration (as opposed to, say, other hooks). - btrfs_command = hook_config.get('btrfs_command', 'btrfs') - findmnt_command = hook_config.get('findmnt_command', 'findmnt') - subvolumes = get_subvolumes(btrfs_command, findmnt_command, patterns) + # Based on the configured patterns, determine Btrfs subvolumes to backup. Only consider those + # patterns that came from actual user configuration (as opposed to, say, other hooks). + btrfs_command = hook_config.get('btrfs_command', 'btrfs') + findmnt_command = hook_config.get('findmnt_command', 'findmnt') + subvolumes = get_subvolumes(btrfs_command, findmnt_command, patterns) - if not subvolumes: - logger.warning(f'No Btrfs subvolumes found to snapshot{dry_run_label}') + if not subvolumes: + logger.warning(f'No Btrfs subvolumes found to snapshot{dry_run_label}') - # Snapshot each subvolume, rewriting patterns to use their snapshot paths. - for subvolume in subvolumes: - logger.debug(f'Creating Btrfs snapshot for {subvolume.path} subvolume') + # Snapshot each subvolume, rewriting patterns to use their snapshot paths. + for subvolume in subvolumes: + logger.debug(f'Creating Btrfs snapshot for {subvolume.path} subvolume') - snapshot_path = make_snapshot_path(subvolume.path) + snapshot_path = make_snapshot_path(subvolume.path) - if dry_run: - continue + if dry_run: + continue - snapshot_subvolume(btrfs_command, subvolume.path, snapshot_path) + snapshot_subvolume(btrfs_command, subvolume.path, snapshot_path) - for pattern in subvolume.contained_patterns: - snapshot_pattern = make_borg_snapshot_pattern(subvolume.path, pattern) + for pattern in subvolume.contained_patterns: + snapshot_pattern = make_borg_snapshot_pattern(subvolume.path, pattern) - # Attempt to update the pattern in place, since pattern order matters to Borg. - try: - patterns[patterns.index(pattern)] = snapshot_pattern - except ValueError: - patterns.append(snapshot_pattern) + # Attempt to update the pattern in place, since pattern order matters to Borg. + try: + patterns[patterns.index(pattern)] = snapshot_pattern + except ValueError: + patterns.append(snapshot_pattern) - patterns.append(make_snapshot_exclude_pattern(subvolume.path)) + patterns.append(make_snapshot_exclude_pattern(subvolume.path)) - return [] + return [] def delete_snapshot(btrfs_command, snapshot_path): # pragma: no cover diff --git a/borgmatic/hooks/data_source/lvm.py b/borgmatic/hooks/data_source/lvm.py index 11cd1c09..b5ea7beb 100644 --- a/borgmatic/hooks/data_source/lvm.py +++ b/borgmatic/hooks/data_source/lvm.py @@ -10,6 +10,7 @@ import subprocess import borgmatic.borg.pattern import borgmatic.config.paths import borgmatic.execute +import borgmatic.hooks.command import borgmatic.hooks.data_source.snapshot logger = logging.getLogger(__name__) @@ -197,77 +198,84 @@ def dump_data_sources( If this is a dry run, then don't actually snapshot anything. ''' - dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' - logger.info(f'Snapshotting LVM logical volumes{dry_run_label}') + with borgmatic.hooks.command.Before_after_hooks( + command_hooks=config.get('commands'), + function_name='dump_data_sources', + hook_name='lvm', + umask=config.get('umask'), + dry_run=dry_run, + ): + dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' + logger.info(f'Snapshotting LVM logical volumes{dry_run_label}') - # List logical volumes to get their mount points, but only consider those patterns that came - # from actual user configuration (as opposed to, say, other hooks). - lsblk_command = hook_config.get('lsblk_command', 'lsblk') - requested_logical_volumes = get_logical_volumes(lsblk_command, patterns) + # List logical volumes to get their mount points, but only consider those patterns that came + # from actual user configuration (as opposed to, say, other hooks). + lsblk_command = hook_config.get('lsblk_command', 'lsblk') + requested_logical_volumes = get_logical_volumes(lsblk_command, patterns) - # Snapshot each logical volume, rewriting source directories to use the snapshot paths. - snapshot_suffix = f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}' - normalized_runtime_directory = os.path.normpath(borgmatic_runtime_directory) + # Snapshot each logical volume, rewriting source directories to use the snapshot paths. + snapshot_suffix = f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}' + normalized_runtime_directory = os.path.normpath(borgmatic_runtime_directory) - if not requested_logical_volumes: - logger.warning(f'No LVM logical volumes found to snapshot{dry_run_label}') + if not requested_logical_volumes: + logger.warning(f'No LVM logical volumes found to snapshot{dry_run_label}') - for logical_volume in requested_logical_volumes: - snapshot_name = f'{logical_volume.name}_{snapshot_suffix}' - logger.debug( - f'Creating LVM snapshot {snapshot_name} of {logical_volume.mount_point}{dry_run_label}' - ) - - if not dry_run: - snapshot_logical_volume( - hook_config.get('lvcreate_command', 'lvcreate'), - snapshot_name, - logical_volume.device_path, - hook_config.get('snapshot_size', DEFAULT_SNAPSHOT_SIZE), + for logical_volume in requested_logical_volumes: + snapshot_name = f'{logical_volume.name}_{snapshot_suffix}' + logger.debug( + f'Creating LVM snapshot {snapshot_name} of {logical_volume.mount_point}{dry_run_label}' ) - # Get the device path for the snapshot we just created. - try: - snapshot = get_snapshots( - hook_config.get('lvs_command', 'lvs'), snapshot_name=snapshot_name - )[0] - except IndexError: - raise ValueError(f'Cannot find LVM snapshot {snapshot_name}') + if not dry_run: + snapshot_logical_volume( + hook_config.get('lvcreate_command', 'lvcreate'), + snapshot_name, + logical_volume.device_path, + hook_config.get('snapshot_size', DEFAULT_SNAPSHOT_SIZE), + ) - # Mount the snapshot into a particular named temporary directory so that the snapshot ends - # up in the Borg archive at the "original" logical volume mount point path. - snapshot_mount_path = os.path.join( - normalized_runtime_directory, - 'lvm_snapshots', - hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest( - MOUNT_POINT_HASH_LENGTH - ), - logical_volume.mount_point.lstrip(os.path.sep), - ) - - logger.debug( - f'Mounting LVM snapshot {snapshot_name} at {snapshot_mount_path}{dry_run_label}' - ) - - if dry_run: - continue - - mount_snapshot( - hook_config.get('mount_command', 'mount'), snapshot.device_path, snapshot_mount_path - ) - - for pattern in logical_volume.contained_patterns: - snapshot_pattern = make_borg_snapshot_pattern( - pattern, logical_volume, normalized_runtime_directory - ) - - # Attempt to update the pattern in place, since pattern order matters to Borg. + # Get the device path for the snapshot we just created. try: - patterns[patterns.index(pattern)] = snapshot_pattern - except ValueError: - patterns.append(snapshot_pattern) + snapshot = get_snapshots( + hook_config.get('lvs_command', 'lvs'), snapshot_name=snapshot_name + )[0] + except IndexError: + raise ValueError(f'Cannot find LVM snapshot {snapshot_name}') - return [] + # Mount the snapshot into a particular named temporary directory so that the snapshot ends + # up in the Borg archive at the "original" logical volume mount point path. + snapshot_mount_path = os.path.join( + normalized_runtime_directory, + 'lvm_snapshots', + hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest( + MOUNT_POINT_HASH_LENGTH + ), + logical_volume.mount_point.lstrip(os.path.sep), + ) + + logger.debug( + f'Mounting LVM snapshot {snapshot_name} at {snapshot_mount_path}{dry_run_label}' + ) + + if dry_run: + continue + + mount_snapshot( + hook_config.get('mount_command', 'mount'), snapshot.device_path, snapshot_mount_path + ) + + for pattern in logical_volume.contained_patterns: + snapshot_pattern = make_borg_snapshot_pattern( + pattern, logical_volume, normalized_runtime_directory + ) + + # Attempt to update the pattern in place, since pattern order matters to Borg. + try: + patterns[patterns.index(pattern)] = snapshot_pattern + except ValueError: + patterns.append(snapshot_pattern) + + return [] def unmount_snapshot(umount_command, snapshot_mount_path): # pragma: no cover diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index 1f140793..de5309c7 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -6,6 +6,7 @@ import shlex import borgmatic.borg.pattern import borgmatic.config.paths +import borgmatic.hooks.command import borgmatic.hooks.credential.parse from borgmatic.execute import ( execute_command, @@ -242,71 +243,78 @@ def dump_data_sources( Also append the the parent directory of the database dumps to the given patterns list, so the dumps actually get backed up. ''' - dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' - processes = [] + with borgmatic.hooks.command.Before_after_hooks( + command_hooks=config.get('commands'), + before_after='dump_data_sources', + hook_name='mariadb', + umask=config.get('umask'), + dry_run=dry_run, + ): + dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' + processes = [] - logger.info(f'Dumping MariaDB databases{dry_run_label}') + logger.info(f'Dumping MariaDB databases{dry_run_label}') - for database in databases: - dump_path = make_dump_path(borgmatic_runtime_directory) - username = borgmatic.hooks.credential.parse.resolve_credential( - database.get('username'), config - ) - password = borgmatic.hooks.credential.parse.resolve_credential( - database.get('password'), config - ) - environment = dict(os.environ) - dump_database_names = database_names_to_dump( - database, config, username, password, environment, dry_run - ) + for database in databases: + dump_path = make_dump_path(borgmatic_runtime_directory) + username = borgmatic.hooks.credential.parse.resolve_credential( + database.get('username'), config + ) + password = borgmatic.hooks.credential.parse.resolve_credential( + database.get('password'), config + ) + environment = dict(os.environ) + dump_database_names = database_names_to_dump( + database, config, username, password, environment, dry_run + ) - if not dump_database_names: - if dry_run: - continue + if not dump_database_names: + if dry_run: + continue - raise ValueError('Cannot find any MariaDB databases to dump.') + raise ValueError('Cannot find any MariaDB databases to dump.') - if database['name'] == 'all' and database.get('format'): - for dump_name in dump_database_names: - renamed_database = copy.copy(database) - renamed_database['name'] = dump_name + if database['name'] == 'all' and database.get('format'): + for dump_name in dump_database_names: + renamed_database = copy.copy(database) + renamed_database['name'] = dump_name + processes.append( + execute_dump_command( + renamed_database, + config, + username, + password, + dump_path, + (dump_name,), + environment, + dry_run, + dry_run_label, + ) + ) + else: processes.append( execute_dump_command( - renamed_database, + database, config, username, password, dump_path, - (dump_name,), + dump_database_names, environment, dry_run, dry_run_label, ) ) - else: - processes.append( - execute_dump_command( - database, - config, - username, - password, - dump_path, - dump_database_names, - environment, - dry_run, - dry_run_label, + + if not dry_run: + patterns.append( + borgmatic.borg.pattern.Pattern( + os.path.join(borgmatic_runtime_directory, 'mariadb_databases'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) ) - if not dry_run: - patterns.append( - borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'mariadb_databases'), - source=borgmatic.borg.pattern.Pattern_source.HOOK, - ) - ) - - return [process for process in processes if process] + return [process for process in processes if process] def remove_data_source_dumps( diff --git a/borgmatic/hooks/data_source/mongodb.py b/borgmatic/hooks/data_source/mongodb.py index 37cc63dd..a40ce41b 100644 --- a/borgmatic/hooks/data_source/mongodb.py +++ b/borgmatic/hooks/data_source/mongodb.py @@ -4,6 +4,7 @@ import shlex import borgmatic.borg.pattern import borgmatic.config.paths +import borgmatic.hooks.command import borgmatic.hooks.credential.parse from borgmatic.execute import execute_command, execute_command_with_processes from borgmatic.hooks.data_source import dump @@ -48,45 +49,53 @@ def dump_data_sources( Also append the the parent directory of the database dumps to the given patterns list, so the dumps actually get backed up. ''' - dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' + with borgmatic.hooks.command.Before_after_hooks( + command_hooks=config.get('commands'), + before_after='dump_data_sources', + hook_name='mongodb', + umask=config.get('umask'), + dry_run=dry_run, + ): + dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' - logger.info(f'Dumping MongoDB databases{dry_run_label}') + logger.info(f'Dumping MongoDB databases{dry_run_label}') - processes = [] - for database in databases: - name = database['name'] - dump_filename = dump.make_data_source_dump_filename( - make_dump_path(borgmatic_runtime_directory), - name, - database.get('hostname'), - database.get('port'), - ) - dump_format = database.get('format', 'archive') + processes = [] - logger.debug( - f'Dumping MongoDB database {name} to {dump_filename}{dry_run_label}', - ) - if dry_run: - continue - - command = build_dump_command(database, config, dump_filename, dump_format) - - if dump_format == 'directory': - dump.create_parent_directory_for_dump(dump_filename) - execute_command(command, shell=True) - else: - dump.create_named_pipe_for_dump(dump_filename) - processes.append(execute_command(command, shell=True, run_to_completion=False)) - - if not dry_run: - patterns.append( - borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'mongodb_databases'), - source=borgmatic.borg.pattern.Pattern_source.HOOK, + for database in databases: + name = database['name'] + dump_filename = dump.make_data_source_dump_filename( + make_dump_path(borgmatic_runtime_directory), + name, + database.get('hostname'), + database.get('port'), ) - ) + dump_format = database.get('format', 'archive') - return processes + logger.debug( + f'Dumping MongoDB database {name} to {dump_filename}{dry_run_label}', + ) + if dry_run: + continue + + command = build_dump_command(database, config, dump_filename, dump_format) + + if dump_format == 'directory': + dump.create_parent_directory_for_dump(dump_filename) + execute_command(command, shell=True) + else: + dump.create_named_pipe_for_dump(dump_filename) + processes.append(execute_command(command, shell=True, run_to_completion=False)) + + if not dry_run: + patterns.append( + borgmatic.borg.pattern.Pattern( + os.path.join(borgmatic_runtime_directory, 'mongodb_databases'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, + ) + ) + + return processes def make_password_config_file(password): diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index 7c5f84f2..def98d90 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -5,6 +5,7 @@ import shlex import borgmatic.borg.pattern import borgmatic.config.paths +import borgmatic.hooks.command import borgmatic.hooks.credential.parse import borgmatic.hooks.data_source.mariadb from borgmatic.execute import ( @@ -169,71 +170,78 @@ def dump_data_sources( Also append the the parent directory of the database dumps to the given patterns list, so the dumps actually get backed up. ''' - dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' - processes = [] + with borgmatic.hooks.command.Before_after_hooks( + command_hooks=config.get('commands'), + before_after='dump_data_sources', + hook_name='mysql', + umask=config.get('umask'), + dry_run=dry_run, + ): + dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' + processes = [] - logger.info(f'Dumping MySQL databases{dry_run_label}') + logger.info(f'Dumping MySQL databases{dry_run_label}') - for database in databases: - dump_path = make_dump_path(borgmatic_runtime_directory) - username = borgmatic.hooks.credential.parse.resolve_credential( - database.get('username'), config - ) - password = borgmatic.hooks.credential.parse.resolve_credential( - database.get('password'), config - ) - environment = dict(os.environ) - dump_database_names = database_names_to_dump( - database, config, username, password, environment, dry_run - ) + for database in databases: + dump_path = make_dump_path(borgmatic_runtime_directory) + username = borgmatic.hooks.credential.parse.resolve_credential( + database.get('username'), config + ) + password = borgmatic.hooks.credential.parse.resolve_credential( + database.get('password'), config + ) + environment = dict(os.environ) + dump_database_names = database_names_to_dump( + database, config, username, password, environment, dry_run + ) - if not dump_database_names: - if dry_run: - continue + if not dump_database_names: + if dry_run: + continue - raise ValueError('Cannot find any MySQL databases to dump.') + raise ValueError('Cannot find any MySQL databases to dump.') - if database['name'] == 'all' and database.get('format'): - for dump_name in dump_database_names: - renamed_database = copy.copy(database) - renamed_database['name'] = dump_name + if database['name'] == 'all' and database.get('format'): + for dump_name in dump_database_names: + renamed_database = copy.copy(database) + renamed_database['name'] = dump_name + processes.append( + execute_dump_command( + renamed_database, + config, + username, + password, + dump_path, + (dump_name,), + environment, + dry_run, + dry_run_label, + ) + ) + else: processes.append( execute_dump_command( - renamed_database, + database, config, username, password, dump_path, - (dump_name,), + dump_database_names, environment, dry_run, dry_run_label, ) ) - else: - processes.append( - execute_dump_command( - database, - config, - username, - password, - dump_path, - dump_database_names, - environment, - dry_run, - dry_run_label, + + if not dry_run: + patterns.append( + borgmatic.borg.pattern.Pattern( + os.path.join(borgmatic_runtime_directory, 'mysql_databases'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) ) - if not dry_run: - patterns.append( - borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'mysql_databases'), - source=borgmatic.borg.pattern.Pattern_source.HOOK, - ) - ) - - return [process for process in processes if process] + return [process for process in processes if process] def remove_data_source_dumps( diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index 4f470c52..99628706 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -7,6 +7,7 @@ import shlex import borgmatic.borg.pattern import borgmatic.config.paths +import borgmatic.hooks.command import borgmatic.hooks.credential.parse from borgmatic.execute import ( execute_command, @@ -141,112 +142,127 @@ def dump_data_sources( Raise ValueError if the databases to dump cannot be determined. ''' - dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' - processes = [] + with borgmatic.hooks.command.Before_after_hooks( + command_hooks=config.get('commands'), + before_after='dump_data_sources', + hook_name='postgresql', + umask=config.get('umask'), + dry_run=dry_run, + ): + dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' + processes = [] - logger.info(f'Dumping PostgreSQL databases{dry_run_label}') + logger.info(f'Dumping PostgreSQL databases{dry_run_label}') - for database in databases: - environment = make_environment(database, config) - dump_path = make_dump_path(borgmatic_runtime_directory) - dump_database_names = database_names_to_dump(database, config, environment, dry_run) + for database in databases: + environment = make_environment(database, config) + dump_path = make_dump_path(borgmatic_runtime_directory) + dump_database_names = database_names_to_dump(database, config, environment, dry_run) - if not dump_database_names: - if dry_run: - continue + if not dump_database_names: + if dry_run: + continue - raise ValueError('Cannot find any PostgreSQL databases to dump.') + raise ValueError('Cannot find any PostgreSQL databases to dump.') - for database_name in dump_database_names: - dump_format = database.get('format', None if database_name == 'all' else 'custom') - compression = database.get('compression') - default_dump_command = 'pg_dumpall' if database_name == 'all' else 'pg_dump' - dump_command = tuple( - shlex.quote(part) - for part in shlex.split(database.get('pg_dump_command') or default_dump_command) - ) - dump_filename = dump.make_data_source_dump_filename( - dump_path, - database_name, - database.get('hostname'), - database.get('port'), - ) - if os.path.exists(dump_filename): - logger.warning( - f'Skipping duplicate dump of PostgreSQL database "{database_name}" to {dump_filename}' + for database_name in dump_database_names: + dump_format = database.get('format', None if database_name == 'all' else 'custom') + compression = database.get('compression') + default_dump_command = 'pg_dumpall' if database_name == 'all' else 'pg_dump' + dump_command = tuple( + shlex.quote(part) + for part in shlex.split(database.get('pg_dump_command') or default_dump_command) ) - continue - - command = ( - dump_command - + ( - '--no-password', - '--clean', - '--if-exists', + dump_filename = dump.make_data_source_dump_filename( + dump_path, + database_name, + database.get('hostname'), + database.get('port'), ) - + (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ()) - + (('--port', shlex.quote(str(database['port']))) if 'port' in database else ()) - + ( - ( - '--username', - shlex.quote( - borgmatic.hooks.credential.parse.resolve_credential( - database['username'], config - ) - ), + if os.path.exists(dump_filename): + logger.warning( + f'Skipping duplicate dump of PostgreSQL database "{database_name}" to {dump_filename}' ) - if 'username' in database - else () - ) - + (('--no-owner',) if database.get('no_owner', False) else ()) - + (('--format', shlex.quote(dump_format)) if dump_format else ()) - + (('--compress', shlex.quote(str(compression))) if compression is not None else ()) - + (('--file', shlex.quote(dump_filename)) if dump_format == 'directory' else ()) - + ( - tuple(shlex.quote(option) for option in database['options'].split(' ')) - if 'options' in database - else () - ) - + (() if database_name == 'all' else (shlex.quote(database_name),)) - # Use shell redirection rather than the --file flag to sidestep synchronization issues - # when pg_dump/pg_dumpall tries to write to a named pipe. But for the directory dump - # format in a particular, a named destination is required, and redirection doesn't work. - + (('>', shlex.quote(dump_filename)) if dump_format != 'directory' else ()) - ) + continue - logger.debug( - f'Dumping PostgreSQL database "{database_name}" to {dump_filename}{dry_run_label}' - ) - if dry_run: - continue - - if dump_format == 'directory': - dump.create_parent_directory_for_dump(dump_filename) - execute_command( - command, - shell=True, - environment=environment, + command = ( + dump_command + + ( + '--no-password', + '--clean', + '--if-exists', + ) + + ( + ('--host', shlex.quote(database['hostname'])) + if 'hostname' in database + else () + ) + + (('--port', shlex.quote(str(database['port']))) if 'port' in database else ()) + + ( + ( + '--username', + shlex.quote( + borgmatic.hooks.credential.parse.resolve_credential( + database['username'], config + ) + ), + ) + if 'username' in database + else () + ) + + (('--no-owner',) if database.get('no_owner', False) else ()) + + (('--format', shlex.quote(dump_format)) if dump_format else ()) + + ( + ('--compress', shlex.quote(str(compression))) + if compression is not None + else () + ) + + (('--file', shlex.quote(dump_filename)) if dump_format == 'directory' else ()) + + ( + tuple(shlex.quote(option) for option in database['options'].split(' ')) + if 'options' in database + else () + ) + + (() if database_name == 'all' else (shlex.quote(database_name),)) + # Use shell redirection rather than the --file flag to sidestep synchronization issues + # when pg_dump/pg_dumpall tries to write to a named pipe. But for the directory dump + # format in a particular, a named destination is required, and redirection doesn't work. + + (('>', shlex.quote(dump_filename)) if dump_format != 'directory' else ()) ) - else: - dump.create_named_pipe_for_dump(dump_filename) - processes.append( + + logger.debug( + f'Dumping PostgreSQL database "{database_name}" to {dump_filename}{dry_run_label}' + ) + if dry_run: + continue + + if dump_format == 'directory': + dump.create_parent_directory_for_dump(dump_filename) execute_command( command, shell=True, environment=environment, - run_to_completion=False, ) + else: + dump.create_named_pipe_for_dump(dump_filename) + processes.append( + execute_command( + command, + shell=True, + environment=environment, + run_to_completion=False, + ) + ) + + if not dry_run: + patterns.append( + borgmatic.borg.pattern.Pattern( + os.path.join(borgmatic_runtime_directory, 'postgresql_databases'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) - - if not dry_run: - patterns.append( - borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'postgresql_databases'), - source=borgmatic.borg.pattern.Pattern_source.HOOK, ) - ) - return processes + return processes def remove_data_source_dumps( diff --git a/borgmatic/hooks/data_source/sqlite.py b/borgmatic/hooks/data_source/sqlite.py index b36c0ccb..01e5b471 100644 --- a/borgmatic/hooks/data_source/sqlite.py +++ b/borgmatic/hooks/data_source/sqlite.py @@ -4,6 +4,7 @@ import shlex import borgmatic.borg.pattern import borgmatic.config.paths +import borgmatic.hooks.command from borgmatic.execute import execute_command, execute_command_with_processes from borgmatic.hooks.data_source import dump @@ -47,55 +48,62 @@ def dump_data_sources( Also append the the parent directory of the database dumps to the given patterns list, so the dumps actually get backed up. ''' - dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' - processes = [] + with borgmatic.hooks.command.Before_after_hooks( + command_hooks=config.get('commands'), + before_after='dump_data_sources', + hook_name='sqlite', + umask=config.get('umask'), + dry_run=dry_run, + ): + dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' + processes = [] - logger.info(f'Dumping SQLite databases{dry_run_label}') + logger.info(f'Dumping SQLite databases{dry_run_label}') - for database in databases: - database_path = database['path'] + for database in databases: + database_path = database['path'] - if database['name'] == 'all': - logger.warning('The "all" database name has no meaning for SQLite databases') - if not os.path.exists(database_path): - logger.warning( - f'No SQLite database at {database_path}; an empty database will be created and dumped' + if database['name'] == 'all': + logger.warning('The "all" database name has no meaning for SQLite databases') + if not os.path.exists(database_path): + logger.warning( + f'No SQLite database at {database_path}; an empty database will be created and dumped' + ) + + dump_path = make_dump_path(borgmatic_runtime_directory) + dump_filename = dump.make_data_source_dump_filename(dump_path, database['name']) + + if os.path.exists(dump_filename): + logger.warning( + f'Skipping duplicate dump of SQLite database at {database_path} to {dump_filename}' + ) + continue + + command = ( + 'sqlite3', + shlex.quote(database_path), + '.dump', + '>', + shlex.quote(dump_filename), + ) + logger.debug( + f'Dumping SQLite database at {database_path} to {dump_filename}{dry_run_label}' + ) + if dry_run: + continue + + dump.create_named_pipe_for_dump(dump_filename) + processes.append(execute_command(command, shell=True, run_to_completion=False)) + + if not dry_run: + patterns.append( + borgmatic.borg.pattern.Pattern( + os.path.join(borgmatic_runtime_directory, 'sqlite_databases'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, + ) ) - dump_path = make_dump_path(borgmatic_runtime_directory) - dump_filename = dump.make_data_source_dump_filename(dump_path, database['name']) - - if os.path.exists(dump_filename): - logger.warning( - f'Skipping duplicate dump of SQLite database at {database_path} to {dump_filename}' - ) - continue - - command = ( - 'sqlite3', - shlex.quote(database_path), - '.dump', - '>', - shlex.quote(dump_filename), - ) - logger.debug( - f'Dumping SQLite database at {database_path} to {dump_filename}{dry_run_label}' - ) - if dry_run: - continue - - dump.create_named_pipe_for_dump(dump_filename) - processes.append(execute_command(command, shell=True, run_to_completion=False)) - - if not dry_run: - patterns.append( - borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'sqlite_databases'), - source=borgmatic.borg.pattern.Pattern_source.HOOK, - ) - ) - - return processes + return processes def remove_data_source_dumps( diff --git a/borgmatic/hooks/data_source/zfs.py b/borgmatic/hooks/data_source/zfs.py index 9cb77587..0f09f25f 100644 --- a/borgmatic/hooks/data_source/zfs.py +++ b/borgmatic/hooks/data_source/zfs.py @@ -9,6 +9,7 @@ import subprocess import borgmatic.borg.pattern import borgmatic.config.paths import borgmatic.execute +import borgmatic.hooks.command import borgmatic.hooks.data_source.snapshot logger = logging.getLogger(__name__) @@ -243,64 +244,71 @@ def dump_data_sources( If this is a dry run, then don't actually snapshot anything. ''' - dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' - logger.info(f'Snapshotting ZFS datasets{dry_run_label}') + with borgmatic.hooks.command.Before_after_hooks( + command_hooks=config.get('commands'), + before_after='dump_data_sources', + hook_name='zfs', + umask=config.get('umask'), + dry_run=dry_run, + ): + dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' + logger.info(f'Snapshotting ZFS datasets{dry_run_label}') - # List ZFS datasets to get their mount points, but only consider those patterns that came from - # actual user configuration (as opposed to, say, other hooks). - zfs_command = hook_config.get('zfs_command', 'zfs') - requested_datasets = get_datasets_to_backup(zfs_command, patterns) + # List ZFS datasets to get their mount points, but only consider those patterns that came from + # actual user configuration (as opposed to, say, other hooks). + zfs_command = hook_config.get('zfs_command', 'zfs') + requested_datasets = get_datasets_to_backup(zfs_command, patterns) - # Snapshot each dataset, rewriting patterns to use the snapshot paths. - snapshot_name = f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}' - normalized_runtime_directory = os.path.normpath(borgmatic_runtime_directory) + # Snapshot each dataset, rewriting patterns to use the snapshot paths. + snapshot_name = f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}' + normalized_runtime_directory = os.path.normpath(borgmatic_runtime_directory) - if not requested_datasets: - logger.warning(f'No ZFS datasets found to snapshot{dry_run_label}') + if not requested_datasets: + logger.warning(f'No ZFS datasets found to snapshot{dry_run_label}') - for dataset in requested_datasets: - full_snapshot_name = f'{dataset.name}@{snapshot_name}' - logger.debug( - f'Creating ZFS snapshot {full_snapshot_name} of {dataset.mount_point}{dry_run_label}' - ) - - if not dry_run: - snapshot_dataset(zfs_command, full_snapshot_name) - - # Mount the snapshot into a particular named temporary directory so that the snapshot ends - # up in the Borg archive at the "original" dataset mount point path. - snapshot_mount_path = os.path.join( - normalized_runtime_directory, - 'zfs_snapshots', - hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest( - MOUNT_POINT_HASH_LENGTH - ), - dataset.mount_point.lstrip(os.path.sep), - ) - - logger.debug( - f'Mounting ZFS snapshot {full_snapshot_name} at {snapshot_mount_path}{dry_run_label}' - ) - - if dry_run: - continue - - mount_snapshot( - hook_config.get('mount_command', 'mount'), full_snapshot_name, snapshot_mount_path - ) - - for pattern in dataset.contained_patterns: - snapshot_pattern = make_borg_snapshot_pattern( - pattern, dataset, normalized_runtime_directory + for dataset in requested_datasets: + full_snapshot_name = f'{dataset.name}@{snapshot_name}' + logger.debug( + f'Creating ZFS snapshot {full_snapshot_name} of {dataset.mount_point}{dry_run_label}' ) - # Attempt to update the pattern in place, since pattern order matters to Borg. - try: - patterns[patterns.index(pattern)] = snapshot_pattern - except ValueError: - patterns.append(snapshot_pattern) + if not dry_run: + snapshot_dataset(zfs_command, full_snapshot_name) - return [] + # Mount the snapshot into a particular named temporary directory so that the snapshot ends + # up in the Borg archive at the "original" dataset mount point path. + snapshot_mount_path = os.path.join( + normalized_runtime_directory, + 'zfs_snapshots', + hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest( + MOUNT_POINT_HASH_LENGTH + ), + dataset.mount_point.lstrip(os.path.sep), + ) + + logger.debug( + f'Mounting ZFS snapshot {full_snapshot_name} at {snapshot_mount_path}{dry_run_label}' + ) + + if dry_run: + continue + + mount_snapshot( + hook_config.get('mount_command', 'mount'), full_snapshot_name, snapshot_mount_path + ) + + for pattern in dataset.contained_patterns: + snapshot_pattern = make_borg_snapshot_pattern( + pattern, dataset, normalized_runtime_directory + ) + + # Attempt to update the pattern in place, since pattern order matters to Borg. + try: + patterns[patterns.index(pattern)] = snapshot_pattern + except ValueError: + patterns.append(snapshot_pattern) + + return [] def unmount_snapshot(umount_command, snapshot_mount_path): # pragma: no cover diff --git a/borgmatic/hooks/dispatch.py b/borgmatic/hooks/dispatch.py index 970d5d5d..e12e70a0 100644 --- a/borgmatic/hooks/dispatch.py +++ b/borgmatic/hooks/dispatch.py @@ -63,20 +63,7 @@ def call_hook(function_name, config, hook_name, *args, **kwargs): logger.debug(f'Calling {hook_name} hook function {function_name}') - borgmatic.hooks.command.execute_hooks( - borgmatic.hooks.command.filter_hooks(config.get('commands'), before=function_name, hook_name=hook_name), - config.get('umask'), - dry_run=False, # FIXME: Need to get this from somewhere. - ) - - try: - return getattr(module, function_name)(hook_config, config, *args, **kwargs) - finally: - borgmatic.hooks.command.execute_hooks( - borgmatic.hooks.command.filter_hooks(config.get('commands'), after=function_name, hook_name=hook_name), - config.get('umask'), - dry_run=False, # FIXME: Need to get this from somewhere. - ) + return getattr(module, function_name)(hook_config, config, *args, **kwargs) def call_hooks(function_name, config, hook_type, *args, **kwargs): From 6a96a78cf1392164000949c223f2af7eb9a01238 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 7 Mar 2025 22:58:25 -0800 Subject: [PATCH 086/226] Fix existing tests (#1019). --- borgmatic/commands/borgmatic.py | 837 +++++++++--------- borgmatic/config/generate.py | 15 +- borgmatic/config/normalize.py | 1 + borgmatic/config/schema.yaml | 8 +- borgmatic/hooks/command.py | 32 +- borgmatic/hooks/data_source/bootstrap.py | 2 +- borgmatic/hooks/data_source/btrfs.py | 2 +- borgmatic/hooks/data_source/lvm.py | 2 +- borgmatic/hooks/data_source/mariadb.py | 2 +- borgmatic/hooks/data_source/mongodb.py | 2 +- borgmatic/hooks/data_source/mysql.py | 2 +- borgmatic/hooks/data_source/postgresql.py | 2 +- borgmatic/hooks/data_source/sqlite.py | 2 +- borgmatic/hooks/data_source/zfs.py | 2 +- tests/unit/actions/test_check.py | 10 - tests/unit/actions/test_compact.py | 4 - tests/unit/actions/test_create.py | 7 - tests/unit/actions/test_extract.py | 2 - tests/unit/actions/test_prune.py | 4 - tests/unit/commands/test_borgmatic.py | 203 +++-- tests/unit/config/test_generate.py | 4 - .../unit/hooks/data_source/test_bootstrap.py | 7 + tests/unit/hooks/data_source/test_btrfs.py | 18 + tests/unit/hooks/data_source/test_lvm.py | 21 + tests/unit/hooks/data_source/test_mariadb.py | 18 + tests/unit/hooks/data_source/test_mongodb.py | 21 + tests/unit/hooks/data_source/test_mysql.py | 18 + .../unit/hooks/data_source/test_postgresql.py | 42 + tests/unit/hooks/data_source/test_sqlite.py | 18 + tests/unit/hooks/data_source/test_zfs.py | 15 + tests/unit/hooks/test_command.py | 72 +- 31 files changed, 799 insertions(+), 596 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 5b9527b8..ddd3fcb4 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -96,135 +96,39 @@ def run_configuration(config_filename, config, config_paths, arguments): f"Skipping {'/'.join(skip_actions)} action{'s' if len(skip_actions) > 1 else ''} due to configured skip_actions" ) - command.execute_hooks( - command.filter_hooks( - config.get('commands'), before='configuration', action_names=arguments.keys() - ), - config.get('umask'), - global_arguments.dry_run, - configuration_filename=config_filename, - ) - - try: - local_borg_version = borg_version.local_borg_version(config, local_path) - logger.debug(f'Borg {local_borg_version}') - except (OSError, CalledProcessError, ValueError) as error: - yield from log_error_records(f'{config_filename}: Error getting local Borg version', error) - return - - try: - if monitoring_hooks_are_activated: - dispatch.call_hooks( - 'initialize_monitor', - config, - dispatch.Hook_type.MONITORING, - config_filename, - monitoring_log_level, - global_arguments.dry_run, + with borgmatic.hooks.command.Before_after_hooks( + command_hooks=config.get('commands'), + before_after='configuration', + umask=config.get('umask'), + dry_run=global_arguments.dry_run, + action_names=arguments.keys(), + ): + try: + local_borg_version = borg_version.local_borg_version(config, local_path) + logger.debug(f'Borg {local_borg_version}') + except (OSError, CalledProcessError, ValueError) as error: + yield from log_error_records( + f'{config_filename}: Error getting local Borg version', error ) - - dispatch.call_hooks( - 'ping_monitor', - config, - dispatch.Hook_type.MONITORING, - config_filename, - monitor.State.START, - monitoring_log_level, - global_arguments.dry_run, - ) - except (OSError, CalledProcessError) as error: - if command.considered_soft_failure(error): return - encountered_error = error - yield from log_error_records(f'{config_filename}: Error pinging monitor', error) - - if not encountered_error: - repo_queue = Queue() - for repo in config['repositories']: - repo_queue.put( - (repo, 0), - ) - - while not repo_queue.empty(): - repository, retry_num = repo_queue.get() - - with Log_prefix(repository.get('label', repository['path'])): - logger.debug('Running actions for repository') - timeout = retry_num * retry_wait - if timeout: - logger.warning(f'Sleeping {timeout}s before next retry') - time.sleep(timeout) - try: - yield from run_actions( - arguments=arguments, - config_filename=config_filename, - config=config, - config_paths=config_paths, - local_path=local_path, - remote_path=remote_path, - local_borg_version=local_borg_version, - repository=repository, - ) - except (OSError, CalledProcessError, ValueError) as error: - if retry_num < retries: - repo_queue.put( - (repository, retry_num + 1), - ) - tuple( # Consume the generator so as to trigger logging. - log_error_records( - 'Error running actions for repository', - error, - levelno=logging.WARNING, - log_command_error_output=True, - ) - ) - logger.warning(f'Retrying... attempt {retry_num + 1}/{retries}') - continue - - if command.considered_soft_failure(error): - continue - - yield from log_error_records( - 'Error running actions for repository', - error, - ) - encountered_error = error - error_repository = repository['path'] - - try: - if monitoring_hooks_are_activated: - # Send logs irrespective of error. - dispatch.call_hooks( - 'ping_monitor', - config, - dispatch.Hook_type.MONITORING, - config_filename, - monitor.State.LOG, - monitoring_log_level, - global_arguments.dry_run, - ) - except (OSError, CalledProcessError) as error: - if not command.considered_soft_failure(error): - encountered_error = error - yield from log_error_records('Error pinging monitor', error) - - if not encountered_error: try: if monitoring_hooks_are_activated: + dispatch.call_hooks( + 'initialize_monitor', + config, + dispatch.Hook_type.MONITORING, + config_filename, + monitoring_log_level, + global_arguments.dry_run, + ) + dispatch.call_hooks( 'ping_monitor', config, dispatch.Hook_type.MONITORING, config_filename, - monitor.State.FINISH, - monitoring_log_level, - global_arguments.dry_run, - ) - dispatch.call_hooks( - 'destroy_monitor', - config, - dispatch.Hook_type.MONITORING, + monitor.State.START, monitoring_log_level, global_arguments.dry_run, ) @@ -235,51 +139,138 @@ def run_configuration(config_filename, config, config_paths, arguments): encountered_error = error yield from log_error_records(f'{config_filename}: Error pinging monitor', error) - if encountered_error: - command.execute_hooks( - command.filter_hooks( - config.get('commands'), after='error', action_names=arguments.keys() - ), - config.get('umask'), - global_arguments.dry_run, - configuration_filename=config_filename, - repository=error_repository, - error=encountered_error, - output=getattr(encountered_error, 'output', ''), - ) + if not encountered_error: + repo_queue = Queue() + for repo in config['repositories']: + repo_queue.put( + (repo, 0), + ) - command.execute_hooks( - command.filter_hooks( - config.get('commands'), after='configuration', action_names=arguments.keys() - ), - config.get('umask'), - global_arguments.dry_run, - configuration_filename=config_filename, - ) + while not repo_queue.empty(): + repository, retry_num = repo_queue.get() + + with Log_prefix(repository.get('label', repository['path'])): + logger.debug('Running actions for repository') + timeout = retry_num * retry_wait + if timeout: + logger.warning(f'Sleeping {timeout}s before next retry') + time.sleep(timeout) + try: + yield from run_actions( + arguments=arguments, + config_filename=config_filename, + config=config, + config_paths=config_paths, + local_path=local_path, + remote_path=remote_path, + local_borg_version=local_borg_version, + repository=repository, + ) + except (OSError, CalledProcessError, ValueError) as error: + if retry_num < retries: + repo_queue.put( + (repository, retry_num + 1), + ) + tuple( # Consume the generator so as to trigger logging. + log_error_records( + 'Error running actions for repository', + error, + levelno=logging.WARNING, + log_command_error_output=True, + ) + ) + logger.warning(f'Retrying... attempt {retry_num + 1}/{retries}') + continue + + if command.considered_soft_failure(error): + continue + + yield from log_error_records( + 'Error running actions for repository', + error, + ) + encountered_error = error + error_repository = repository['path'] - if encountered_error and using_primary_action: try: - dispatch.call_hooks( - 'ping_monitor', - config, - dispatch.Hook_type.MONITORING, - config_filename, - monitor.State.FAIL, - monitoring_log_level, - global_arguments.dry_run, - ) - dispatch.call_hooks( - 'destroy_monitor', - config, - dispatch.Hook_type.MONITORING, - monitoring_log_level, - global_arguments.dry_run, - ) + if monitoring_hooks_are_activated: + # Send logs irrespective of error. + dispatch.call_hooks( + 'ping_monitor', + config, + dispatch.Hook_type.MONITORING, + config_filename, + monitor.State.LOG, + monitoring_log_level, + global_arguments.dry_run, + ) except (OSError, CalledProcessError) as error: - if command.considered_soft_failure(error): - return + if not command.considered_soft_failure(error): + encountered_error = error + yield from log_error_records('Error pinging monitor', error) - yield from log_error_records(f'{config_filename}: Error running on-error hook', error) + if not encountered_error: + try: + if monitoring_hooks_are_activated: + dispatch.call_hooks( + 'ping_monitor', + config, + dispatch.Hook_type.MONITORING, + config_filename, + monitor.State.FINISH, + monitoring_log_level, + global_arguments.dry_run, + ) + dispatch.call_hooks( + 'destroy_monitor', + config, + dispatch.Hook_type.MONITORING, + monitoring_log_level, + global_arguments.dry_run, + ) + except (OSError, CalledProcessError) as error: + if command.considered_soft_failure(error): + return + + encountered_error = error + yield from log_error_records(f'{config_filename}: Error pinging monitor', error) + + if encountered_error: + try: + command.execute_hooks( + command.filter_hooks( + config.get('commands'), after='error', action_names=arguments.keys() + ), + config.get('umask'), + global_arguments.dry_run, + configuration_filename=config_filename, + repository=error_repository, + error=encountered_error, + output=getattr(encountered_error, 'output', ''), + ) + dispatch.call_hooks( + 'ping_monitor', + config, + dispatch.Hook_type.MONITORING, + config_filename, + monitor.State.FAIL, + monitoring_log_level, + global_arguments.dry_run, + ) + dispatch.call_hooks( + 'destroy_monitor', + config, + dispatch.Hook_type.MONITORING, + monitoring_log_level, + global_arguments.dry_run, + ) + except (OSError, CalledProcessError) as error: + if command.considered_soft_failure(error): + return + + yield from log_error_records( + f'{config_filename}: Error running after error hook', error + ) def run_actions( @@ -319,256 +310,236 @@ def run_actions( } skip_actions = set(get_skip_actions(config, arguments)) - command.execute_hooks( - command.filter_hooks( - config.get('commands'), before='repository', action_names=arguments.keys() - ), - config.get('umask'), - global_arguments.dry_run, + with borgmatic.hooks.command.Before_after_hooks( + command_hooks=config.get('commands'), + before_after='repository', + umask=config.get('umask'), + dry_run=global_arguments.dry_run, + action_names=arguments.keys(), **hook_context, - ) + ): + for action_name, action_arguments in arguments.items(): + if action_name == 'global': + continue - for action_name, action_arguments in arguments.items(): - if action_name == 'global': - continue - - command.execute_hooks( - command.filter_hooks( - config.get('commands'), before='action', action_names=arguments.keys() - ), - config.get('umask'), - global_arguments.dry_run, - **hook_context, - ) - - if action_name == 'repo-create' and action_name not in skip_actions: - borgmatic.actions.repo_create.run_repo_create( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'transfer' and action_name not in skip_actions: - borgmatic.actions.transfer.run_transfer( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'create' and action_name not in skip_actions: - yield from borgmatic.actions.create.run_create( - config_filename, - repository, - config, - config_paths, - local_borg_version, - action_arguments, - global_arguments, - dry_run_label, - local_path, - remote_path, - ) - elif action_name == 'prune' and action_name not in skip_actions: - borgmatic.actions.prune.run_prune( - config_filename, - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - dry_run_label, - local_path, - remote_path, - ) - elif action_name == 'compact' and action_name not in skip_actions: - borgmatic.actions.compact.run_compact( - config_filename, - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - dry_run_label, - local_path, - remote_path, - ) - elif action_name == 'check' and action_name not in skip_actions: - if checks.repository_enabled_for_checks(repository, config): - borgmatic.actions.check.run_check( - config_filename, - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'extract' and action_name not in skip_actions: - borgmatic.actions.extract.run_extract( - config_filename, - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'export-tar' and action_name not in skip_actions: - borgmatic.actions.export_tar.run_export_tar( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'mount' and action_name not in skip_actions: - borgmatic.actions.mount.run_mount( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'restore' and action_name not in skip_actions: - borgmatic.actions.restore.run_restore( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'repo-list' and action_name not in skip_actions: - yield from borgmatic.actions.repo_list.run_repo_list( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'list' and action_name not in skip_actions: - yield from borgmatic.actions.list.run_list( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'repo-info' and action_name not in skip_actions: - yield from borgmatic.actions.repo_info.run_repo_info( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'info' and action_name not in skip_actions: - yield from borgmatic.actions.info.run_info( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'break-lock' and action_name not in skip_actions: - borgmatic.actions.break_lock.run_break_lock( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'export' and action_name not in skip_actions: - borgmatic.actions.export_key.run_export_key( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'change-passphrase' and action_name not in skip_actions: - borgmatic.actions.change_passphrase.run_change_passphrase( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'delete' and action_name not in skip_actions: - borgmatic.actions.delete.run_delete( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'repo-delete' and action_name not in skip_actions: - borgmatic.actions.repo_delete.run_repo_delete( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'borg' and action_name not in skip_actions: - borgmatic.actions.borg.run_borg( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - - command.execute_hooks( - command.filter_hooks( - config.get('commands'), after='action', action_names=arguments.keys() - ), - config.get('umask'), - global_arguments.dry_run, - **hook_context, - ) - - command.execute_hooks( - command.filter_hooks( - config.get('commands'), after='repository', action_names=arguments.keys() - ), - config.get('umask'), - global_arguments.dry_run, - **hook_context, - ) + with borgmatic.hooks.command.Before_after_hooks( + command_hooks=config.get('commands'), + before_after='action', + umask=config.get('umask'), + dry_run=global_arguments.dry_run, + action_names=arguments.keys(), + **hook_context, + ): + if action_name == 'repo-create' and action_name not in skip_actions: + borgmatic.actions.repo_create.run_repo_create( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'transfer' and action_name not in skip_actions: + borgmatic.actions.transfer.run_transfer( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'create' and action_name not in skip_actions: + yield from borgmatic.actions.create.run_create( + config_filename, + repository, + config, + config_paths, + local_borg_version, + action_arguments, + global_arguments, + dry_run_label, + local_path, + remote_path, + ) + elif action_name == 'prune' and action_name not in skip_actions: + borgmatic.actions.prune.run_prune( + config_filename, + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + dry_run_label, + local_path, + remote_path, + ) + elif action_name == 'compact' and action_name not in skip_actions: + borgmatic.actions.compact.run_compact( + config_filename, + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + dry_run_label, + local_path, + remote_path, + ) + elif action_name == 'check' and action_name not in skip_actions: + if checks.repository_enabled_for_checks(repository, config): + borgmatic.actions.check.run_check( + config_filename, + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'extract' and action_name not in skip_actions: + borgmatic.actions.extract.run_extract( + config_filename, + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'export-tar' and action_name not in skip_actions: + borgmatic.actions.export_tar.run_export_tar( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'mount' and action_name not in skip_actions: + borgmatic.actions.mount.run_mount( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'restore' and action_name not in skip_actions: + borgmatic.actions.restore.run_restore( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'repo-list' and action_name not in skip_actions: + yield from borgmatic.actions.repo_list.run_repo_list( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'list' and action_name not in skip_actions: + yield from borgmatic.actions.list.run_list( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'repo-info' and action_name not in skip_actions: + yield from borgmatic.actions.repo_info.run_repo_info( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'info' and action_name not in skip_actions: + yield from borgmatic.actions.info.run_info( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'break-lock' and action_name not in skip_actions: + borgmatic.actions.break_lock.run_break_lock( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'export' and action_name not in skip_actions: + borgmatic.actions.export_key.run_export_key( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'change-passphrase' and action_name not in skip_actions: + borgmatic.actions.change_passphrase.run_change_passphrase( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'delete' and action_name not in skip_actions: + borgmatic.actions.delete.run_delete( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'repo-delete' and action_name not in skip_actions: + borgmatic.actions.repo_delete.run_repo_delete( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'borg' and action_name not in skip_actions: + borgmatic.actions.borg.run_borg( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) def load_configurations(config_filenames, overrides=None, resolve_env=True): @@ -848,20 +819,19 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): ) return - if 'create' in arguments: - try: - for config_filename, config in configs.items(): - command.execute_hooks( - command.filter_hooks( - config.get('commands'), before='everything', action_names=arguments.keys() - ), - config.get('umask'), - arguments['global'].dry_run, - configuration_filename=config_filename, - ) - except (CalledProcessError, ValueError, OSError) as error: - yield from log_error_records('Error running pre-everything hook', error) - return + try: + for config_filename, config in configs.items(): + command.execute_hooks( + command.filter_hooks( + config.get('commands'), before='everything', action_names=arguments.keys() + ), + config.get('umask'), + arguments['global'].dry_run, + configuration_filename=config_filename, + ) + except (CalledProcessError, ValueError, OSError) as error: + yield from log_error_records('Error running pre-everything hook', error) + return # Execute the actions corresponding to each configuration file. json_results = [] @@ -901,19 +871,18 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): if json_results: sys.stdout.write(json.dumps(json_results)) - if 'create' in arguments: - try: - for config_filename, config in configs.items(): - command.execute_hooks( - command.filter_hooks( - config.get('commands'), after='everything', action_names=arguments.keys() - ), - config.get('umask'), - arguments['global'].dry_run, - configuration_filename=config_filename, - ) - except (CalledProcessError, ValueError, OSError) as error: - yield from log_error_records('Error running post-everything hook', error) + try: + for config_filename, config in configs.items(): + command.execute_hooks( + command.filter_hooks( + config.get('commands'), after='everything', action_names=arguments.keys() + ), + config.get('umask'), + arguments['global'].dry_run, + configuration_filename=config_filename, + ) + except (CalledProcessError, ValueError, OSError) as error: + yield from log_error_records('Error running post-everything hook', error) def exit_with_help_link(): # pragma: no cover diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index d3096717..58fd03a7 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -43,10 +43,13 @@ def get_properties(schema): return schema['properties'] -def schema_to_sample_configuration(schema, source_config, level=0, parent_is_sequence=False): +def schema_to_sample_configuration(schema, source_config=None, level=0, parent_is_sequence=False): ''' Given a loaded configuration schema and a source configuration, generate and return sample config for the schema. Include comments for each option based on the schema "description". + + If a source config is given, walk it alongside the given schema so that both can be taken into + account when commenting out particular options in add_comments_to_configuration_object(). ''' schema_type = schema.get('type') example = schema.get('example') @@ -72,7 +75,7 @@ def schema_to_sample_configuration(schema, source_config, level=0, parent_is_seq ( field_name, schema_to_sample_configuration( - sub_schema, source_config.get(field_name, {}), level + 1 + sub_schema, (source_config or {}).get(field_name, {}), level + 1 ), ) for field_name, sub_schema in get_properties(schema).items() @@ -204,7 +207,9 @@ DEFAULT_KEYS = {'source_directories', 'repositories', 'keep_daily'} COMMENTED_OUT_SENTINEL = 'COMMENT_OUT' -def add_comments_to_configuration_object(config, schema, source_config, indent=0, skip_first=False): +def add_comments_to_configuration_object( + config, schema, source_config=None, indent=0, skip_first=False +): ''' Using descriptions from a schema as a source, add those descriptions as comments to the given configuration dict, putting them before each field. Indent the comment the given number of @@ -224,7 +229,9 @@ def add_comments_to_configuration_object(config, schema, source_config, indent=0 # If this isn't a default key, add an indicator to the comment flagging it to be commented # out from the sample configuration. This sentinel is consumed by downstream processing that # does the actual commenting out. - if field_name not in DEFAULT_KEYS and field_name not in source_config: + if field_name not in DEFAULT_KEYS and ( + source_config is None or field_name not in source_config + ): description = ( '\n'.join((description, COMMENTED_OUT_SENTINEL)) if description diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index 7da0023d..d8244297 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -134,6 +134,7 @@ def normalize_commands(config_filename, config): config.setdefault('commands', []).append( { preposition: 'everything', + 'when': ['create'], 'run': commands, } ) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index e46656c1..24609f40 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1015,7 +1015,7 @@ properties: List of actions for which the commands will be run. Defaults to running for all actions. Ignored for "dump_data_sources", which by its nature only - runs for particular actions. + runs for "create". example: [create, prune, compact, check] run: type: array @@ -1091,8 +1091,10 @@ properties: - key - borg description: | - List of actions for which the commands will be run. - Defaults to running for all actions. + List of actions for which the commands will be + run. Defaults to running for all actions. Ignored + for "dump_data_sources", which by its nature only + runs for "create". example: [create, prune, compact, check] run: type: array diff --git a/borgmatic/hooks/command.py b/borgmatic/hooks/command.py index d46184c1..ab4d8af0 100644 --- a/borgmatic/hooks/command.py +++ b/borgmatic/hooks/command.py @@ -138,9 +138,9 @@ class Before_after_hooks: with borgmatic.hooks.command.Before_after_hooks( command_hooks=config.get('commands'), before_after='do_stuff', - hook_name='myhook', umask=config.get('umask'), dry_run=dry_run, + hook_name='myhook', ): do() some() @@ -150,17 +150,27 @@ class Before_after_hooks: and "after" command hooks execute after the wrapped code completes. ''' - def __init__(self, command_hooks, before_after, hook_name, umask, dry_run, **context): + def __init__( + self, + command_hooks, + before_after, + umask, + dry_run, + hook_name=None, + action_names=None, + **context, + ): ''' - Given a sequence of command hook configuration dicts, the before/after name, the name of the - calling hook, a umask to run commands with, a dry run flag, and any context for the executed - commands, save those data points for use below. + Given a sequence of command hook configuration dicts, the before/after name, a umask to run + commands with, a dry run flag, the name of the calling hook, a sequence of action names, and + any context for the executed commands, save those data points for use below. ''' self.command_hooks = command_hooks self.before_after = before_after - self.hook_name = hook_name self.umask = umask self.dry_run = dry_run + self.hook_name = hook_name + self.action_names = action_names self.context = context def __enter__(self): @@ -169,7 +179,10 @@ class Before_after_hooks: ''' execute_hooks( borgmatic.hooks.command.filter_hooks( - self.command_hooks, before=self.before_after, hook_name=self.hook_name + self.command_hooks, + before=self.before_after, + hook_name=self.hook_name, + action_names=self.action_names, ), self.umask, self.dry_run, @@ -182,7 +195,10 @@ class Before_after_hooks: ''' execute_hooks( borgmatic.hooks.command.filter_hooks( - self.command_hooks, after=self.before_after, hook_name=self.hook_name + self.command_hooks, + after=self.before_after, + hook_name=self.hook_name, + action_names=self.action_names, ), self.umask, self.dry_run, diff --git a/borgmatic/hooks/data_source/bootstrap.py b/borgmatic/hooks/data_source/bootstrap.py index 5a2b4f09..e992cc2d 100644 --- a/borgmatic/hooks/data_source/bootstrap.py +++ b/borgmatic/hooks/data_source/bootstrap.py @@ -41,9 +41,9 @@ def dump_data_sources( with borgmatic.hooks.command.Before_after_hooks( command_hooks=config.get('commands'), before_after='dump_data_sources', - hook_name='bootstrap', umask=config.get('umask'), dry_run=dry_run, + hook_name='bootstrap', ): borgmatic_manifest_path = os.path.join( borgmatic_runtime_directory, 'bootstrap', 'manifest.json' diff --git a/borgmatic/hooks/data_source/btrfs.py b/borgmatic/hooks/data_source/btrfs.py index 7023fea6..5d6e3116 100644 --- a/borgmatic/hooks/data_source/btrfs.py +++ b/borgmatic/hooks/data_source/btrfs.py @@ -208,9 +208,9 @@ def dump_data_sources( with borgmatic.hooks.command.Before_after_hooks( command_hooks=config.get('commands'), before_after='dump_data_sources', - hook_name='btrfs', umask=config.get('umask'), dry_run=dry_run, + hook_name='btrfs', ): dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' logger.info(f'Snapshotting Btrfs subvolumes{dry_run_label}') diff --git a/borgmatic/hooks/data_source/lvm.py b/borgmatic/hooks/data_source/lvm.py index b5ea7beb..7ee07296 100644 --- a/borgmatic/hooks/data_source/lvm.py +++ b/borgmatic/hooks/data_source/lvm.py @@ -201,9 +201,9 @@ def dump_data_sources( with borgmatic.hooks.command.Before_after_hooks( command_hooks=config.get('commands'), function_name='dump_data_sources', - hook_name='lvm', umask=config.get('umask'), dry_run=dry_run, + hook_name='lvm', ): dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' logger.info(f'Snapshotting LVM logical volumes{dry_run_label}') diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index de5309c7..3881d933 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -246,9 +246,9 @@ def dump_data_sources( with borgmatic.hooks.command.Before_after_hooks( command_hooks=config.get('commands'), before_after='dump_data_sources', - hook_name='mariadb', umask=config.get('umask'), dry_run=dry_run, + hook_name='mariadb', ): dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' processes = [] diff --git a/borgmatic/hooks/data_source/mongodb.py b/borgmatic/hooks/data_source/mongodb.py index a40ce41b..033fa4ab 100644 --- a/borgmatic/hooks/data_source/mongodb.py +++ b/borgmatic/hooks/data_source/mongodb.py @@ -52,9 +52,9 @@ def dump_data_sources( with borgmatic.hooks.command.Before_after_hooks( command_hooks=config.get('commands'), before_after='dump_data_sources', - hook_name='mongodb', umask=config.get('umask'), dry_run=dry_run, + hook_name='mongodb', ): dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index def98d90..351a575f 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -173,9 +173,9 @@ def dump_data_sources( with borgmatic.hooks.command.Before_after_hooks( command_hooks=config.get('commands'), before_after='dump_data_sources', - hook_name='mysql', umask=config.get('umask'), dry_run=dry_run, + hook_name='mysql', ): dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' processes = [] diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index 99628706..b25704f7 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -145,9 +145,9 @@ def dump_data_sources( with borgmatic.hooks.command.Before_after_hooks( command_hooks=config.get('commands'), before_after='dump_data_sources', - hook_name='postgresql', umask=config.get('umask'), dry_run=dry_run, + hook_name='postgresql', ): dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' processes = [] diff --git a/borgmatic/hooks/data_source/sqlite.py b/borgmatic/hooks/data_source/sqlite.py index 01e5b471..8c4b6008 100644 --- a/borgmatic/hooks/data_source/sqlite.py +++ b/borgmatic/hooks/data_source/sqlite.py @@ -51,9 +51,9 @@ def dump_data_sources( with borgmatic.hooks.command.Before_after_hooks( command_hooks=config.get('commands'), before_after='dump_data_sources', - hook_name='sqlite', umask=config.get('umask'), dry_run=dry_run, + hook_name='sqlite', ): dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' processes = [] diff --git a/borgmatic/hooks/data_source/zfs.py b/borgmatic/hooks/data_source/zfs.py index 0f09f25f..c7073140 100644 --- a/borgmatic/hooks/data_source/zfs.py +++ b/borgmatic/hooks/data_source/zfs.py @@ -247,9 +247,9 @@ def dump_data_sources( with borgmatic.hooks.command.Before_after_hooks( command_hooks=config.get('commands'), before_after='dump_data_sources', - hook_name='zfs', umask=config.get('umask'), dry_run=dry_run, + hook_name='zfs', ): dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' logger.info(f'Snapshotting ZFS datasets{dry_run_label}') diff --git a/tests/unit/actions/test_check.py b/tests/unit/actions/test_check.py index 90b716b5..6b16a901 100644 --- a/tests/unit/actions/test_check.py +++ b/tests/unit/actions/test_check.py @@ -1405,7 +1405,6 @@ def test_run_check_checks_archives_for_configured_repository(): flexmock(module).should_receive('make_check_time_path') flexmock(module).should_receive('write_check_time') flexmock(module.borgmatic.borg.extract).should_receive('extract_last_archive_dry_run').never() - flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) check_arguments = flexmock( repository=None, progress=flexmock(), @@ -1419,7 +1418,6 @@ def test_run_check_checks_archives_for_configured_repository(): config_filename='test.yaml', repository={'path': 'repo'}, config={'repositories': ['repo']}, - hook_context={}, local_borg_version=None, check_arguments=check_arguments, global_arguments=global_arguments, @@ -1441,7 +1439,6 @@ def test_run_check_runs_configured_extract_check(): flexmock(module.borgmatic.borg.extract).should_receive('extract_last_archive_dry_run').once() flexmock(module).should_receive('make_check_time_path') flexmock(module).should_receive('write_check_time') - flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) check_arguments = flexmock( repository=None, progress=flexmock(), @@ -1455,7 +1452,6 @@ def test_run_check_runs_configured_extract_check(): config_filename='test.yaml', repository={'path': 'repo'}, config={'repositories': ['repo']}, - hook_context={}, local_borg_version=None, check_arguments=check_arguments, global_arguments=global_arguments, @@ -1480,7 +1476,6 @@ def test_run_check_runs_configured_spot_check(): flexmock(module.borgmatic.actions.check).should_receive('spot_check').once() flexmock(module).should_receive('make_check_time_path') flexmock(module).should_receive('write_check_time') - flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) check_arguments = flexmock( repository=None, progress=flexmock(), @@ -1494,7 +1489,6 @@ def test_run_check_runs_configured_spot_check(): config_filename='test.yaml', repository={'path': 'repo'}, config={'repositories': ['repo']}, - hook_context={}, local_borg_version=None, check_arguments=check_arguments, global_arguments=global_arguments, @@ -1516,7 +1510,6 @@ def test_run_check_without_checks_runs_nothing_except_hooks(): flexmock(module).should_receive('make_check_time_path') flexmock(module).should_receive('write_check_time').never() flexmock(module.borgmatic.borg.extract).should_receive('extract_last_archive_dry_run').never() - flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) check_arguments = flexmock( repository=None, progress=flexmock(), @@ -1530,7 +1523,6 @@ def test_run_check_without_checks_runs_nothing_except_hooks(): config_filename='test.yaml', repository={'path': 'repo'}, config={'repositories': ['repo']}, - hook_context={}, local_borg_version=None, check_arguments=check_arguments, global_arguments=global_arguments, @@ -1569,7 +1561,6 @@ def test_run_check_checks_archives_in_selected_repository(): config_filename='test.yaml', repository={'path': 'repo'}, config={'repositories': ['repo']}, - hook_context={}, local_borg_version=None, check_arguments=check_arguments, global_arguments=global_arguments, @@ -1597,7 +1588,6 @@ def test_run_check_bails_if_repository_does_not_match(): config_filename='test.yaml', repository={'path': 'repo'}, config={'repositories': ['repo']}, - hook_context={}, local_borg_version=None, check_arguments=check_arguments, global_arguments=global_arguments, diff --git a/tests/unit/actions/test_compact.py b/tests/unit/actions/test_compact.py index 0df83fdd..14dcfaad 100644 --- a/tests/unit/actions/test_compact.py +++ b/tests/unit/actions/test_compact.py @@ -8,7 +8,6 @@ def test_compact_actions_calls_hooks_for_configured_repository(): flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never() flexmock(module.borgmatic.borg.compact).should_receive('compact_segments').once() - flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) compact_arguments = flexmock( repository=None, progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock() ) @@ -18,7 +17,6 @@ def test_compact_actions_calls_hooks_for_configured_repository(): config_filename='test.yaml', repository={'path': 'repo'}, config={}, - hook_context={}, local_borg_version=None, compact_arguments=compact_arguments, global_arguments=global_arguments, @@ -44,7 +42,6 @@ def test_compact_runs_with_selected_repository(): config_filename='test.yaml', repository={'path': 'repo'}, config={}, - hook_context={}, local_borg_version=None, compact_arguments=compact_arguments, global_arguments=global_arguments, @@ -70,7 +67,6 @@ def test_compact_bails_if_repository_does_not_match(): config_filename='test.yaml', repository={'path': 'repo'}, config={}, - hook_context={}, local_borg_version=None, compact_arguments=compact_arguments, global_arguments=global_arguments, diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 5e4a4c9f..1e93557d 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -424,7 +424,6 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository(): flexmock() ) flexmock(module.borgmatic.borg.create).should_receive('create_archive').once() - flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({}) flexmock(module.borgmatic.hooks.dispatch).should_receive( 'call_hooks_even_if_unconfigured' @@ -447,7 +446,6 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository(): repository={'path': 'repo'}, config={}, config_paths=['/tmp/test.yaml'], - hook_context={}, local_borg_version=None, create_arguments=create_arguments, global_arguments=global_arguments, @@ -467,7 +465,6 @@ def test_run_create_runs_with_selected_repository(): flexmock() ) flexmock(module.borgmatic.borg.create).should_receive('create_archive').once() - flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({}) flexmock(module.borgmatic.hooks.dispatch).should_receive( 'call_hooks_even_if_unconfigured' @@ -490,7 +487,6 @@ def test_run_create_runs_with_selected_repository(): repository={'path': 'repo'}, config={}, config_paths=['/tmp/test.yaml'], - hook_context={}, local_borg_version=None, create_arguments=create_arguments, global_arguments=global_arguments, @@ -523,7 +519,6 @@ def test_run_create_bails_if_repository_does_not_match(): repository='repo', config={}, config_paths=['/tmp/test.yaml'], - hook_context={}, local_borg_version=None, create_arguments=create_arguments, global_arguments=global_arguments, @@ -547,7 +542,6 @@ def test_run_create_produces_json(): ) parsed_json = flexmock() flexmock(module.borgmatic.actions.json).should_receive('parse_json').and_return(parsed_json) - flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({}) flexmock(module.borgmatic.hooks.dispatch).should_receive( 'call_hooks_even_if_unconfigured' @@ -570,7 +564,6 @@ def test_run_create_produces_json(): repository={'path': 'repo'}, config={}, config_paths=['/tmp/test.yaml'], - hook_context={}, local_borg_version=None, create_arguments=create_arguments, global_arguments=global_arguments, diff --git a/tests/unit/actions/test_extract.py b/tests/unit/actions/test_extract.py index 7fadf4d7..c483adcc 100644 --- a/tests/unit/actions/test_extract.py +++ b/tests/unit/actions/test_extract.py @@ -7,7 +7,6 @@ def test_run_extract_calls_hooks(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) flexmock(module.borgmatic.borg.extract).should_receive('extract_archive') - flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) extract_arguments = flexmock( paths=flexmock(), progress=flexmock(), @@ -22,7 +21,6 @@ def test_run_extract_calls_hooks(): config_filename='test.yaml', repository={'path': 'repo'}, config={'repositories': ['repo']}, - hook_context={}, local_borg_version=None, extract_arguments=extract_arguments, global_arguments=global_arguments, diff --git a/tests/unit/actions/test_prune.py b/tests/unit/actions/test_prune.py index d5dd182e..762a83c2 100644 --- a/tests/unit/actions/test_prune.py +++ b/tests/unit/actions/test_prune.py @@ -7,7 +7,6 @@ def test_run_prune_calls_hooks_for_configured_repository(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never() flexmock(module.borgmatic.borg.prune).should_receive('prune_archives').once() - flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) prune_arguments = flexmock(repository=None, stats=flexmock(), list_archives=flexmock()) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -15,7 +14,6 @@ def test_run_prune_calls_hooks_for_configured_repository(): config_filename='test.yaml', repository={'path': 'repo'}, config={}, - hook_context={}, local_borg_version=None, prune_arguments=prune_arguments, global_arguments=global_arguments, @@ -38,7 +36,6 @@ def test_run_prune_runs_with_selected_repository(): config_filename='test.yaml', repository={'path': 'repo'}, config={}, - hook_context={}, local_borg_version=None, prune_arguments=prune_arguments, global_arguments=global_arguments, @@ -61,7 +58,6 @@ def test_run_prune_bails_if_repository_does_not_match(): config_filename='test.yaml', repository='repo', config={}, - hook_context={}, local_borg_version=None, prune_arguments=prune_arguments, global_arguments=global_arguments, diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 1615dcf4..ce87cb6b 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -30,6 +30,7 @@ def test_get_skip_actions_uses_config_and_arguments(config, arguments, expected_ def test_run_configuration_runs_actions_for_each_repository(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) expected_results = [flexmock(), flexmock()] flexmock(module).should_receive('Log_prefix').and_return(flexmock()) @@ -37,7 +38,7 @@ def test_run_configuration_runs_actions_for_each_repository(): expected_results[1:] ) config = {'repositories': [{'path': 'foo'}, {'path': 'bar'}]} - arguments = {'global': flexmock(monitoring_verbosity=1)} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False)} results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -47,11 +48,12 @@ def test_run_configuration_runs_actions_for_each_repository(): def test_run_configuration_with_skip_actions_does_not_raise(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return(['compact']) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_return(flexmock()).and_return(flexmock()) config = {'repositories': [{'path': 'foo'}, {'path': 'bar'}], 'skip_actions': ['compact']} - arguments = {'global': flexmock(monitoring_verbosity=1)} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False)} list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -59,8 +61,8 @@ def test_run_configuration_with_skip_actions_does_not_raise(): def test_run_configuration_with_invalid_borg_version_errors(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_raise(ValueError) - flexmock(module.command).should_receive('execute_hook').never() flexmock(module.dispatch).should_receive('call_hooks').never() flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').never() @@ -73,6 +75,7 @@ def test_run_configuration_with_invalid_borg_version_errors(): def test_run_configuration_logs_monitor_start_error(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.dispatch).should_receive('call_hooks').and_raise(OSError).and_return( None @@ -81,6 +84,8 @@ def test_run_configuration_logs_monitor_start_error(): flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').never() + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} @@ -92,6 +97,7 @@ def test_run_configuration_logs_monitor_start_error(): def test_run_configuration_bails_for_monitor_start_soft_failure(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') flexmock(module.dispatch).should_receive('call_hooks').and_raise(error).and_return(None) @@ -109,13 +115,15 @@ def test_run_configuration_bails_for_monitor_start_soft_failure(): def test_run_configuration_logs_actions_error(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.command).should_receive('execute_hook') flexmock(module.dispatch).should_receive('call_hooks') expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError) + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False)} @@ -127,6 +135,7 @@ def test_run_configuration_logs_actions_error(): def test_run_configuration_skips_remaining_actions_for_actions_soft_failure_but_still_runs_next_repository_actions(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.dispatch).should_receive('call_hooks').times(5) error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') @@ -146,6 +155,7 @@ def test_run_configuration_skips_remaining_actions_for_actions_soft_failure_but_ def test_run_configuration_logs_monitor_log_error(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return( None @@ -154,6 +164,8 @@ def test_run_configuration_logs_monitor_log_error(): flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_return([]) + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} @@ -165,6 +177,7 @@ def test_run_configuration_logs_monitor_log_error(): def test_run_configuration_still_pings_monitor_for_monitor_log_soft_failure(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return( @@ -185,6 +198,7 @@ def test_run_configuration_still_pings_monitor_for_monitor_log_soft_failure(): def test_run_configuration_logs_monitor_finish_error(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return( None @@ -193,6 +207,8 @@ def test_run_configuration_logs_monitor_finish_error(): flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_return([]) + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} @@ -204,6 +220,7 @@ def test_run_configuration_logs_monitor_finish_error(): def test_run_configuration_bails_for_monitor_finish_soft_failure(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return( @@ -224,6 +241,7 @@ def test_run_configuration_bails_for_monitor_finish_soft_failure(): def test_run_configuration_does_not_call_monitoring_hooks_if_monitoring_hooks_are_disabled(): flexmock(module).should_receive('verbosity_to_log_level').and_return(module.DISABLED) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.dispatch).should_receive('call_hooks').never() @@ -239,8 +257,10 @@ def test_run_configuration_does_not_call_monitoring_hooks_if_monitoring_hooks_ar def test_run_configuration_logs_on_error_hook_error(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.command).should_receive('execute_hook').and_raise(OSError) + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks').and_raise(OSError) expected_results = [flexmock(), flexmock()] flexmock(module).should_receive('log_error_records').and_return( expected_results[:1] @@ -258,9 +278,11 @@ def test_run_configuration_logs_on_error_hook_error(): def test_run_configuration_bails_for_on_error_hook_soft_failure(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') - flexmock(module.command).should_receive('execute_hook').and_raise(error) + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks').and_raise(error) expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('Log_prefix').and_return(flexmock()) @@ -277,14 +299,18 @@ def test_run_configuration_retries_soft_error(): # Run action first fails, second passes. flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError).and_return([]) flexmock(module).should_receive('log_error_records').and_return([flexmock()]).once() + flexmock(module.command).should_receive('filter_hooks').never() + flexmock(module.command).should_receive('execute_hooks').never() config = {'repositories': [{'path': 'foo'}], 'retries': 1} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) + assert results == [] @@ -292,8 +318,8 @@ def test_run_configuration_retries_hard_error(): # Run action fails twice. flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError).times(2) flexmock(module).should_receive('log_error_records').with_args( @@ -307,17 +333,21 @@ def test_run_configuration_retries_hard_error(): 'Error running actions for repository', OSError, ).and_return(error_logs) + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') config = {'repositories': [{'path': 'foo'}], 'retries': 1} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) + assert results == error_logs -def test_run_configuration_repos_ordered(): +def test_run_configuration_retries_repositories_in_order(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError).times(2) expected_results = [flexmock(), flexmock()] @@ -327,17 +357,21 @@ def test_run_configuration_repos_ordered(): flexmock(module).should_receive('log_error_records').with_args( 'Error running actions for repository', OSError ).and_return(expected_results[1:]).ordered() + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') config = {'repositories': [{'path': 'foo'}, {'path': 'bar'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) + assert results == expected_results def test_run_configuration_retries_round_robin(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) flexmock(module).should_receive('log_error_records').with_args( @@ -360,20 +394,24 @@ def test_run_configuration_retries_round_robin(): flexmock(module).should_receive('log_error_records').with_args( 'Error running actions for repository', OSError ).and_return(bar_error_logs).ordered() + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') config = { 'repositories': [{'path': 'foo'}, {'path': 'bar'}], 'retries': 1, } arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) + assert results == foo_error_logs + bar_error_logs -def test_run_configuration_retries_one_passes(): +def test_run_configuration_with_one_retry(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return( [] @@ -394,20 +432,24 @@ def test_run_configuration_retries_one_passes(): flexmock(module).should_receive('log_error_records').with_args( 'Error running actions for repository', OSError ).and_return(error_logs).ordered() + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') config = { 'repositories': [{'path': 'foo'}, {'path': 'bar'}], 'retries': 1, } arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) + assert results == error_logs -def test_run_configuration_retry_wait(): +def test_run_configuration_with_retry_wait_does_backoff_after_each_retry(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) flexmock(module).should_receive('log_error_records').with_args( @@ -438,21 +480,25 @@ def test_run_configuration_retry_wait(): flexmock(module).should_receive('log_error_records').with_args( 'Error running actions for repository', OSError ).and_return(error_logs).ordered() + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') config = { 'repositories': [{'path': 'foo'}], 'retries': 3, 'retry_wait': 10, } arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) + assert results == error_logs -def test_run_configuration_retries_timeout_multiple_repos(): +def test_run_configuration_with_multiple_repositories_retries_with_timeout(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return( [] @@ -479,20 +525,24 @@ def test_run_configuration_retries_timeout_multiple_repos(): flexmock(module).should_receive('log_error_records').with_args( 'Error running actions for repository', OSError ).and_return(error_logs).ordered() + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') config = { 'repositories': [{'path': 'foo'}, {'path': 'bar'}], 'retries': 1, 'retry_wait': 10, } arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) + assert results == error_logs def test_run_actions_runs_repo_create(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.repo_create).should_receive('run_repo_create').once() tuple( @@ -515,19 +565,13 @@ def test_run_actions_runs_repo_create(): def test_run_actions_adds_label_file_to_hook_context(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) expected = flexmock() flexmock(borgmatic.actions.create).should_receive('run_create').with_args( config_filename=object, repository={'path': 'repo', 'label': 'my repo'}, config={'repositories': []}, config_paths=[], - hook_context={ - 'repository_label': 'my repo', - 'log_file': '', - 'repositories': '', - 'repository': 'repo', - }, local_borg_version=object, create_arguments=object, global_arguments=object, @@ -554,19 +598,13 @@ def test_run_actions_adds_label_file_to_hook_context(): def test_run_actions_adds_log_file_to_hook_context(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) expected = flexmock() flexmock(borgmatic.actions.create).should_receive('run_create').with_args( config_filename=object, repository={'path': 'repo'}, config={'repositories': []}, config_paths=[], - hook_context={ - 'repository_label': '', - 'log_file': 'foo', - 'repositories': '', - 'repository': 'repo', - }, local_borg_version=object, create_arguments=object, global_arguments=object, @@ -593,7 +631,7 @@ def test_run_actions_adds_log_file_to_hook_context(): def test_run_actions_runs_transfer(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.transfer).should_receive('run_transfer').once() tuple( @@ -613,7 +651,7 @@ def test_run_actions_runs_transfer(): def test_run_actions_runs_create(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) expected = flexmock() flexmock(borgmatic.actions.create).should_receive('run_create').and_yield(expected).once() @@ -635,7 +673,7 @@ def test_run_actions_runs_create(): def test_run_actions_with_skip_actions_skips_create(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return(['create']) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.create).should_receive('run_create').never() tuple( @@ -655,7 +693,7 @@ def test_run_actions_with_skip_actions_skips_create(): def test_run_actions_runs_prune(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.prune).should_receive('run_prune').once() tuple( @@ -675,7 +713,7 @@ def test_run_actions_runs_prune(): def test_run_actions_with_skip_actions_skips_prune(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return(['prune']) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.prune).should_receive('run_prune').never() tuple( @@ -695,7 +733,7 @@ def test_run_actions_with_skip_actions_skips_prune(): def test_run_actions_runs_compact(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.compact).should_receive('run_compact').once() tuple( @@ -715,7 +753,7 @@ def test_run_actions_runs_compact(): def test_run_actions_with_skip_actions_skips_compact(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return(['compact']) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.compact).should_receive('run_compact').never() tuple( @@ -735,7 +773,7 @@ def test_run_actions_with_skip_actions_skips_compact(): def test_run_actions_runs_check_when_repository_enabled_for_checks(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.checks).should_receive('repository_enabled_for_checks').and_return(True) flexmock(borgmatic.actions.check).should_receive('run_check').once() @@ -756,7 +794,7 @@ def test_run_actions_runs_check_when_repository_enabled_for_checks(): def test_run_actions_skips_check_when_repository_not_enabled_for_checks(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.checks).should_receive('repository_enabled_for_checks').and_return(False) flexmock(borgmatic.actions.check).should_receive('run_check').never() @@ -777,7 +815,7 @@ def test_run_actions_skips_check_when_repository_not_enabled_for_checks(): def test_run_actions_with_skip_actions_skips_check(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return(['check']) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.checks).should_receive('repository_enabled_for_checks').and_return(True) flexmock(borgmatic.actions.check).should_receive('run_check').never() @@ -798,7 +836,7 @@ def test_run_actions_with_skip_actions_skips_check(): def test_run_actions_runs_extract(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.extract).should_receive('run_extract').once() tuple( @@ -818,7 +856,7 @@ def test_run_actions_runs_extract(): def test_run_actions_runs_export_tar(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.export_tar).should_receive('run_export_tar').once() tuple( @@ -838,7 +876,7 @@ def test_run_actions_runs_export_tar(): def test_run_actions_runs_mount(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.mount).should_receive('run_mount').once() tuple( @@ -858,7 +896,7 @@ def test_run_actions_runs_mount(): def test_run_actions_runs_restore(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.restore).should_receive('run_restore').once() tuple( @@ -878,7 +916,7 @@ def test_run_actions_runs_restore(): def test_run_actions_runs_repo_list(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) expected = flexmock() flexmock(borgmatic.actions.repo_list).should_receive('run_repo_list').and_yield(expected).once() @@ -900,7 +938,7 @@ def test_run_actions_runs_repo_list(): def test_run_actions_runs_list(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) expected = flexmock() flexmock(borgmatic.actions.list).should_receive('run_list').and_yield(expected).once() @@ -922,7 +960,7 @@ def test_run_actions_runs_list(): def test_run_actions_runs_repo_info(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) expected = flexmock() flexmock(borgmatic.actions.repo_info).should_receive('run_repo_info').and_yield(expected).once() @@ -944,7 +982,7 @@ def test_run_actions_runs_repo_info(): def test_run_actions_runs_info(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) expected = flexmock() flexmock(borgmatic.actions.info).should_receive('run_info').and_yield(expected).once() @@ -966,7 +1004,7 @@ def test_run_actions_runs_info(): def test_run_actions_runs_break_lock(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.break_lock).should_receive('run_break_lock').once() tuple( @@ -986,7 +1024,7 @@ def test_run_actions_runs_break_lock(): def test_run_actions_runs_export_key(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.export_key).should_receive('run_export_key').once() tuple( @@ -1006,7 +1044,7 @@ def test_run_actions_runs_export_key(): def test_run_actions_runs_change_passphrase(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.change_passphrase).should_receive('run_change_passphrase').once() tuple( @@ -1029,7 +1067,7 @@ def test_run_actions_runs_change_passphrase(): def test_run_actions_runs_delete(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.delete).should_receive('run_delete').once() tuple( @@ -1049,7 +1087,7 @@ def test_run_actions_runs_delete(): def test_run_actions_runs_repo_delete(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.repo_delete).should_receive('run_repo_delete').once() tuple( @@ -1072,7 +1110,7 @@ def test_run_actions_runs_repo_delete(): def test_run_actions_runs_borg(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.borg).should_receive('run_borg').once() tuple( @@ -1092,7 +1130,7 @@ def test_run_actions_runs_borg(): def test_run_actions_runs_multiple_actions_in_argument_order(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.borg).should_receive('run_borg').once().ordered() flexmock(borgmatic.actions.restore).should_receive('run_restore').once().ordered() @@ -1398,11 +1436,12 @@ def test_collect_highlander_action_summary_logs_error_on_run_validate_failure(): def test_collect_configuration_run_summary_logs_info_for_success(): - flexmock(module.command).should_receive('execute_hook').never() flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return([]) - arguments = {} + arguments = {'global': flexmock(dry_run=False)} logs = tuple( module.collect_configuration_run_summary_logs( @@ -1415,6 +1454,8 @@ def test_collect_configuration_run_summary_logs_info_for_success(): def test_collect_configuration_run_summary_executes_hooks_for_create(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return([]) arguments = {'create': flexmock(), 'global': flexmock(monitoring_verbosity=1, dry_run=False)} @@ -1430,9 +1471,11 @@ def test_collect_configuration_run_summary_executes_hooks_for_create(): def test_collect_configuration_run_summary_logs_info_for_success_with_extract(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return([]) - arguments = {'extract': flexmock(repository='repo')} + arguments = {'extract': flexmock(repository='repo'), 'global': flexmock(dry_run=False)} logs = tuple( module.collect_configuration_run_summary_logs( @@ -1462,9 +1505,11 @@ def test_collect_configuration_run_summary_logs_extract_with_repository_error(): def test_collect_configuration_run_summary_logs_info_for_success_with_mount(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return([]) - arguments = {'mount': flexmock(repository='repo')} + arguments = {'mount': flexmock(repository='repo'), 'global': flexmock(dry_run=False)} logs = tuple( module.collect_configuration_run_summary_logs( @@ -1481,7 +1526,7 @@ def test_collect_configuration_run_summary_logs_mount_with_repository_error(): ) expected_logs = (flexmock(),) flexmock(module).should_receive('log_error_records').and_return(expected_logs) - arguments = {'mount': flexmock(repository='repo')} + arguments = {'mount': flexmock(repository='repo'), 'global': flexmock(dry_run=False)} logs = tuple( module.collect_configuration_run_summary_logs( @@ -1493,6 +1538,9 @@ def test_collect_configuration_run_summary_logs_mount_with_repository_error(): def test_collect_configuration_run_summary_logs_missing_configs_error(): + flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') arguments = {'global': flexmock(config_paths=[])} expected_logs = (flexmock(),) flexmock(module).should_receive('log_error_records').and_return(expected_logs) @@ -1505,7 +1553,9 @@ def test_collect_configuration_run_summary_logs_missing_configs_error(): def test_collect_configuration_run_summary_logs_pre_hook_error(): - flexmock(module.command).should_receive('execute_hook').and_raise(ValueError) + flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks').and_raise(ValueError) expected_logs = (flexmock(),) flexmock(module).should_receive('log_error_records').and_return(expected_logs) arguments = {'create': flexmock(), 'global': flexmock(monitoring_verbosity=1, dry_run=False)} @@ -1520,8 +1570,9 @@ def test_collect_configuration_run_summary_logs_pre_hook_error(): def test_collect_configuration_run_summary_logs_post_hook_error(): - flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise(ValueError) flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks').and_return(None).and_raise(ValueError) flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return([]) expected_logs = (flexmock(),) @@ -1543,7 +1594,10 @@ def test_collect_configuration_run_summary_logs_for_list_with_archive_and_reposi ) expected_logs = (flexmock(),) flexmock(module).should_receive('log_error_records').and_return(expected_logs) - arguments = {'list': flexmock(repository='repo', archive='test')} + arguments = { + 'list': flexmock(repository='repo', archive='test'), + 'global': flexmock(dry_run=False), + } logs = tuple( module.collect_configuration_run_summary_logs( @@ -1556,9 +1610,14 @@ def test_collect_configuration_run_summary_logs_for_list_with_archive_and_reposi def test_collect_configuration_run_summary_logs_info_for_success_with_list(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return([]) - arguments = {'list': flexmock(repository='repo', archive=None)} + arguments = { + 'list': flexmock(repository='repo', archive=None), + 'global': flexmock(dry_run=False), + } logs = tuple( module.collect_configuration_run_summary_logs( @@ -1571,12 +1630,14 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_list(): def test_collect_configuration_run_summary_logs_run_configuration_error(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return( [logging.makeLogRecord(dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg='Error'))] ) flexmock(module).should_receive('log_error_records').and_return([]) - arguments = {} + arguments = {'global': flexmock(dry_run=False)} logs = tuple( module.collect_configuration_run_summary_logs( @@ -1589,13 +1650,15 @@ def test_collect_configuration_run_summary_logs_run_configuration_error(): def test_collect_configuration_run_summary_logs_run_umount_error(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return([]) flexmock(module.borg_umount).should_receive('unmount_archive').and_raise(OSError) flexmock(module).should_receive('log_error_records').and_return( [logging.makeLogRecord(dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg='Error'))] ) - arguments = {'umount': flexmock(mount_point='/mnt')} + arguments = {'umount': flexmock(mount_point='/mnt'), 'global': flexmock(dry_run=False)} logs = tuple( module.collect_configuration_run_summary_logs( @@ -1608,6 +1671,8 @@ def test_collect_configuration_run_summary_logs_run_umount_error(): def test_collect_configuration_run_summary_logs_outputs_merged_json_results(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return(['foo', 'bar']).and_return( ['baz'] @@ -1615,7 +1680,7 @@ def test_collect_configuration_run_summary_logs_outputs_merged_json_results(): stdout = flexmock() stdout.should_receive('write').with_args('["foo", "bar", "baz"]').once() flexmock(module.sys).stdout = stdout - arguments = {} + arguments = {'global': flexmock(dry_run=False)} tuple( module.collect_configuration_run_summary_logs( diff --git a/tests/unit/config/test_generate.py b/tests/unit/config/test_generate.py index 530d641c..9b880df6 100644 --- a/tests/unit/config/test_generate.py +++ b/tests/unit/config/test_generate.py @@ -133,7 +133,6 @@ def test_schema_to_sample_configuration_with_unsupported_schema_raises(): def test_merge_source_configuration_into_destination_inserts_map_fields(): destination_config = {'foo': 'dest1', 'bar': 'dest2'} source_config = {'foo': 'source1', 'baz': 'source2'} - flexmock(module).should_receive('remove_commented_out_sentinel') flexmock(module).should_receive('ruamel.yaml.comments.CommentedSeq').replace_with(list) module.merge_source_configuration_into_destination(destination_config, source_config) @@ -144,7 +143,6 @@ def test_merge_source_configuration_into_destination_inserts_map_fields(): def test_merge_source_configuration_into_destination_inserts_nested_map_fields(): destination_config = {'foo': {'first': 'dest1', 'second': 'dest2'}, 'bar': 'dest3'} source_config = {'foo': {'first': 'source1'}} - flexmock(module).should_receive('remove_commented_out_sentinel') flexmock(module).should_receive('ruamel.yaml.comments.CommentedSeq').replace_with(list) module.merge_source_configuration_into_destination(destination_config, source_config) @@ -155,7 +153,6 @@ def test_merge_source_configuration_into_destination_inserts_nested_map_fields() def test_merge_source_configuration_into_destination_inserts_sequence_fields(): destination_config = {'foo': ['dest1', 'dest2'], 'bar': ['dest3'], 'baz': ['dest4']} source_config = {'foo': ['source1'], 'bar': ['source2', 'source3']} - flexmock(module).should_receive('remove_commented_out_sentinel') flexmock(module).should_receive('ruamel.yaml.comments.CommentedSeq').replace_with(list) module.merge_source_configuration_into_destination(destination_config, source_config) @@ -170,7 +167,6 @@ def test_merge_source_configuration_into_destination_inserts_sequence_fields(): def test_merge_source_configuration_into_destination_inserts_sequence_of_maps(): destination_config = {'foo': [{'first': 'dest1', 'second': 'dest2'}], 'bar': 'dest3'} source_config = {'foo': [{'first': 'source1'}, {'other': 'source2'}]} - flexmock(module).should_receive('remove_commented_out_sentinel') flexmock(module).should_receive('ruamel.yaml.comments.CommentedSeq').replace_with(list) module.merge_source_configuration_into_destination(destination_config, source_config) diff --git a/tests/unit/hooks/data_source/test_bootstrap.py b/tests/unit/hooks/data_source/test_bootstrap.py index 1623dfb7..f5299a4e 100644 --- a/tests/unit/hooks/data_source/test_bootstrap.py +++ b/tests/unit/hooks/data_source/test_bootstrap.py @@ -6,6 +6,9 @@ from borgmatic.hooks.data_source import bootstrap as module def test_dump_data_sources_creates_manifest_file(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) flexmock(module.os).should_receive('makedirs') flexmock(module.importlib.metadata).should_receive('version').and_return('1.0.0') @@ -32,6 +35,7 @@ def test_dump_data_sources_creates_manifest_file(): def test_dump_data_sources_with_store_config_files_false_does_not_create_manifest_file(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').never() flexmock(module.os).should_receive('makedirs').never() flexmock(module.json).should_receive('dump').never() hook_config = {'store_config_files': False} @@ -47,6 +51,9 @@ def test_dump_data_sources_with_store_config_files_false_does_not_create_manifes def test_dump_data_sources_with_dry_run_does_not_create_manifest_file(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) flexmock(module.os).should_receive('makedirs').never() flexmock(module.json).should_receive('dump').never() diff --git a/tests/unit/hooks/data_source/test_btrfs.py b/tests/unit/hooks/data_source/test_btrfs.py index 648902c3..1003d1ed 100644 --- a/tests/unit/hooks/data_source/test_btrfs.py +++ b/tests/unit/hooks/data_source/test_btrfs.py @@ -207,6 +207,9 @@ def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_ def test_dump_data_sources_snapshots_each_subvolume_and_updates_patterns(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) patterns = [Pattern('/foo'), Pattern('/mnt/subvol1')] config = {'btrfs': {}} flexmock(module).should_receive('get_subvolumes').and_return( @@ -285,6 +288,9 @@ def test_dump_data_sources_snapshots_each_subvolume_and_updates_patterns(): def test_dump_data_sources_uses_custom_btrfs_command_in_commands(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) patterns = [Pattern('/foo'), Pattern('/mnt/subvol1')] config = {'btrfs': {'btrfs_command': '/usr/local/bin/btrfs'}} flexmock(module).should_receive('get_subvolumes').and_return( @@ -338,6 +344,9 @@ def test_dump_data_sources_uses_custom_btrfs_command_in_commands(): def test_dump_data_sources_uses_custom_findmnt_command_in_commands(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) patterns = [Pattern('/foo'), Pattern('/mnt/subvol1')] config = {'btrfs': {'findmnt_command': '/usr/local/bin/findmnt'}} flexmock(module).should_receive('get_subvolumes').with_args( @@ -393,6 +402,9 @@ def test_dump_data_sources_uses_custom_findmnt_command_in_commands(): def test_dump_data_sources_with_dry_run_skips_snapshot_and_patterns_update(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) patterns = [Pattern('/foo'), Pattern('/mnt/subvol1')] config = {'btrfs': {}} flexmock(module).should_receive('get_subvolumes').and_return( @@ -421,6 +433,9 @@ def test_dump_data_sources_with_dry_run_skips_snapshot_and_patterns_update(): def test_dump_data_sources_without_matching_subvolumes_skips_snapshot_and_patterns_update(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) patterns = [Pattern('/foo'), Pattern('/mnt/subvol1')] config = {'btrfs': {}} flexmock(module).should_receive('get_subvolumes').and_return(()) @@ -445,6 +460,9 @@ def test_dump_data_sources_without_matching_subvolumes_skips_snapshot_and_patter def test_dump_data_sources_snapshots_adds_to_existing_exclude_patterns(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) patterns = [Pattern('/foo'), Pattern('/mnt/subvol1')] config = {'btrfs': {}, 'exclude_patterns': ['/bar']} flexmock(module).should_receive('get_subvolumes').and_return( diff --git a/tests/unit/hooks/data_source/test_lvm.py b/tests/unit/hooks/data_source/test_lvm.py index 34cf5250..77655250 100644 --- a/tests/unit/hooks/data_source/test_lvm.py +++ b/tests/unit/hooks/data_source/test_lvm.py @@ -282,6 +282,9 @@ def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_ def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) config = {'lvm': {}} patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')] logical_volumes = ( @@ -351,6 +354,9 @@ def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns(): def test_dump_data_sources_with_no_logical_volumes_skips_snapshots(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) config = {'lvm': {}} patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')] flexmock(module).should_receive('get_logical_volumes').and_return(()) @@ -373,6 +379,9 @@ def test_dump_data_sources_with_no_logical_volumes_skips_snapshots(): def test_dump_data_sources_uses_snapshot_size_for_snapshot(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) config = {'lvm': {'snapshot_size': '1000PB'}} patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')] logical_volumes = ( @@ -448,6 +457,9 @@ def test_dump_data_sources_uses_snapshot_size_for_snapshot(): def test_dump_data_sources_uses_custom_commands(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) config = { 'lvm': { 'lsblk_command': '/usr/local/bin/lsblk', @@ -534,6 +546,9 @@ def test_dump_data_sources_uses_custom_commands(): def test_dump_data_sources_with_dry_run_skips_snapshots_and_does_not_touch_patterns(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) config = {'lvm': {}} patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')] flexmock(module).should_receive('get_logical_volumes').and_return( @@ -585,6 +600,9 @@ def test_dump_data_sources_with_dry_run_skips_snapshots_and_does_not_touch_patte def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained_patterns(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) config = {'lvm': {}} patterns = [Pattern('/hmm')] logical_volumes = ( @@ -655,6 +673,9 @@ def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained def test_dump_data_sources_with_missing_snapshot_errors(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) config = {'lvm': {}} patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')] flexmock(module).should_receive('get_logical_volumes').and_return( diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py index 871c1969..6f9aaa30 100644 --- a/tests/unit/hooks/data_source/test_mariadb.py +++ b/tests/unit/hooks/data_source/test_mariadb.py @@ -237,6 +237,9 @@ def test_use_streaming_false_for_no_databases(): def test_dump_data_sources_dumps_each_database(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'foo'}, {'name': 'bar'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') @@ -278,6 +281,9 @@ def test_dump_data_sources_dumps_each_database(): def test_dump_data_sources_dumps_with_password(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) database = {'name': 'foo', 'username': 'root', 'password': 'trustsome1'} process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') @@ -312,6 +318,9 @@ def test_dump_data_sources_dumps_with_password(): def test_dump_data_sources_dumps_all_databases_at_once(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'all'}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') @@ -343,6 +352,9 @@ def test_dump_data_sources_dumps_all_databases_at_once(): def test_dump_data_sources_dumps_all_databases_separately_when_format_configured(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'all', 'format': 'sql'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') @@ -850,6 +862,9 @@ def test_execute_dump_command_with_dry_run_skips_mariadb_dump(): def test_dump_data_sources_errors_for_missing_all_databases(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'all'}] flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) @@ -873,6 +888,9 @@ def test_dump_data_sources_errors_for_missing_all_databases(): def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'all'}] flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) diff --git a/tests/unit/hooks/data_source/test_mongodb.py b/tests/unit/hooks/data_source/test_mongodb.py index 0cef73f8..cd74e29f 100644 --- a/tests/unit/hooks/data_source/test_mongodb.py +++ b/tests/unit/hooks/data_source/test_mongodb.py @@ -24,6 +24,9 @@ def test_use_streaming_false_for_no_databases(): def test_dump_data_sources_runs_mongodump_for_each_database(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'foo'}, {'name': 'bar'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') @@ -53,6 +56,9 @@ def test_dump_data_sources_runs_mongodump_for_each_database(): def test_dump_data_sources_with_dry_run_skips_mongodump(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'foo'}, {'name': 'bar'}] flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( @@ -75,6 +81,9 @@ def test_dump_data_sources_with_dry_run_skips_mongodump(): def test_dump_data_sources_runs_mongodump_with_hostname_and_port(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') @@ -111,6 +120,9 @@ def test_dump_data_sources_runs_mongodump_with_hostname_and_port(): def test_dump_data_sources_runs_mongodump_with_username_and_password(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [ { 'name': 'foo', @@ -162,6 +174,9 @@ def test_dump_data_sources_runs_mongodump_with_username_and_password(): def test_dump_data_sources_runs_mongodump_with_directory_format(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'foo', 'format': 'directory'}] flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( @@ -189,6 +204,9 @@ def test_dump_data_sources_runs_mongodump_with_directory_format(): def test_dump_data_sources_runs_mongodump_with_options(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'foo', 'options': '--stuff=such'}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') @@ -222,6 +240,9 @@ def test_dump_data_sources_runs_mongodump_with_options(): def test_dump_data_sources_runs_mongodumpall_for_all_databases(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'all'}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') diff --git a/tests/unit/hooks/data_source/test_mysql.py b/tests/unit/hooks/data_source/test_mysql.py index 63bb7a72..c6d40155 100644 --- a/tests/unit/hooks/data_source/test_mysql.py +++ b/tests/unit/hooks/data_source/test_mysql.py @@ -134,6 +134,9 @@ def test_use_streaming_false_for_no_databases(): def test_dump_data_sources_dumps_each_database(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'foo'}, {'name': 'bar'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') @@ -172,6 +175,9 @@ def test_dump_data_sources_dumps_each_database(): def test_dump_data_sources_dumps_with_password(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) database = {'name': 'foo', 'username': 'root', 'password': 'trustsome1'} process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') @@ -206,6 +212,9 @@ def test_dump_data_sources_dumps_with_password(): def test_dump_data_sources_dumps_all_databases_at_once(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'all'}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') @@ -237,6 +246,9 @@ def test_dump_data_sources_dumps_all_databases_at_once(): def test_dump_data_sources_dumps_all_databases_separately_when_format_configured(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'all', 'format': 'sql'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') @@ -762,6 +774,9 @@ def test_execute_dump_command_with_dry_run_skips_mysqldump(): def test_dump_data_sources_errors_for_missing_all_databases(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'all'}] flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) @@ -785,6 +800,9 @@ def test_dump_data_sources_errors_for_missing_all_databases(): def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'all'}] flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) diff --git a/tests/unit/hooks/data_source/test_postgresql.py b/tests/unit/hooks/data_source/test_postgresql.py index f1589a6e..da73c4e6 100644 --- a/tests/unit/hooks/data_source/test_postgresql.py +++ b/tests/unit/hooks/data_source/test_postgresql.py @@ -236,6 +236,9 @@ def test_use_streaming_false_for_no_databases(): def test_dump_data_sources_runs_pg_dump_for_each_database(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'foo'}, {'name': 'bar'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) @@ -284,6 +287,9 @@ def test_dump_data_sources_runs_pg_dump_for_each_database(): def test_dump_data_sources_raises_when_no_database_names_to_dump(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'foo'}, {'name': 'bar'}] flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') @@ -301,6 +307,9 @@ def test_dump_data_sources_raises_when_no_database_names_to_dump(): def test_dump_data_sources_does_not_raise_when_no_database_names_to_dump(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'foo'}, {'name': 'bar'}] flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') @@ -317,6 +326,9 @@ def test_dump_data_sources_does_not_raise_when_no_database_names_to_dump(): def test_dump_data_sources_with_duplicate_dump_skips_pg_dump(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'foo'}, {'name': 'bar'}] flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') @@ -344,6 +356,9 @@ def test_dump_data_sources_with_duplicate_dump_skips_pg_dump(): def test_dump_data_sources_with_dry_run_skips_pg_dump(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'foo'}, {'name': 'bar'}] flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') @@ -374,6 +389,9 @@ def test_dump_data_sources_with_dry_run_skips_pg_dump(): def test_dump_data_sources_runs_pg_dump_with_hostname_and_port(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] process = flexmock() flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) @@ -420,6 +438,9 @@ def test_dump_data_sources_runs_pg_dump_with_hostname_and_port(): def test_dump_data_sources_runs_pg_dump_with_username_and_password(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1'}] process = flexmock() flexmock(module).should_receive('make_environment').and_return( @@ -466,6 +487,9 @@ def test_dump_data_sources_runs_pg_dump_with_username_and_password(): def test_dump_data_sources_with_username_injection_attack_gets_escaped(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'foo', 'username': 'postgres; naughty-command', 'password': 'trustsome1'}] process = flexmock() flexmock(module).should_receive('make_environment').and_return( @@ -512,6 +536,9 @@ def test_dump_data_sources_with_username_injection_attack_gets_escaped(): def test_dump_data_sources_runs_pg_dump_with_directory_format(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'foo', 'format': 'directory'}] flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') @@ -556,6 +583,9 @@ def test_dump_data_sources_runs_pg_dump_with_directory_format(): def test_dump_data_sources_runs_pg_dump_with_string_compression(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'foo', 'compression': 'winrar'}] processes = [flexmock()] flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) @@ -603,6 +633,9 @@ def test_dump_data_sources_runs_pg_dump_with_string_compression(): def test_dump_data_sources_runs_pg_dump_with_integer_compression(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'foo', 'compression': 0}] processes = [flexmock()] flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) @@ -650,6 +683,9 @@ def test_dump_data_sources_runs_pg_dump_with_integer_compression(): def test_dump_data_sources_runs_pg_dump_with_options(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'foo', 'options': '--stuff=such'}] process = flexmock() flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) @@ -693,6 +729,9 @@ def test_dump_data_sources_runs_pg_dump_with_options(): def test_dump_data_sources_runs_pg_dumpall_for_all_databases(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'all'}] process = flexmock() flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) @@ -725,6 +764,9 @@ def test_dump_data_sources_runs_pg_dumpall_for_all_databases(): def test_dump_data_sources_runs_non_default_pg_dump(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'foo', 'pg_dump_command': 'special_pg_dump --compress *'}] process = flexmock() flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) diff --git a/tests/unit/hooks/data_source/test_sqlite.py b/tests/unit/hooks/data_source/test_sqlite.py index 03afa3c3..6c5a5639 100644 --- a/tests/unit/hooks/data_source/test_sqlite.py +++ b/tests/unit/hooks/data_source/test_sqlite.py @@ -17,6 +17,9 @@ def test_use_streaming_false_for_no_databases(): def test_dump_data_sources_logs_and_skips_if_dump_already_exists(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'path': '/path/to/database', 'name': 'database'}] flexmock(module).should_receive('make_dump_path').and_return('/run/borgmatic') @@ -41,6 +44,9 @@ def test_dump_data_sources_logs_and_skips_if_dump_already_exists(): def test_dump_data_sources_dumps_each_database(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [ {'path': '/path/to/database1', 'name': 'database1'}, {'path': '/path/to/database2', 'name': 'database2'}, @@ -71,6 +77,9 @@ def test_dump_data_sources_dumps_each_database(): def test_dump_data_sources_with_path_injection_attack_gets_escaped(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [ {'path': '/path/to/database1; naughty-command', 'name': 'database1'}, ] @@ -108,6 +117,9 @@ def test_dump_data_sources_with_path_injection_attack_gets_escaped(): def test_dump_data_sources_with_non_existent_path_warns_and_dumps_database(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [ {'path': '/path/to/database1', 'name': 'database1'}, ] @@ -136,6 +148,9 @@ def test_dump_data_sources_with_non_existent_path_warns_and_dumps_database(): def test_dump_data_sources_with_name_all_warns_and_dumps_all_databases(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [ {'path': '/path/to/database1', 'name': 'all'}, ] @@ -166,6 +181,9 @@ def test_dump_data_sources_with_name_all_warns_and_dumps_all_databases(): def test_dump_data_sources_does_not_dump_if_dry_run(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'path': '/path/to/database', 'name': 'database'}] flexmock(module).should_receive('make_dump_path').and_return('/run/borgmatic') diff --git a/tests/unit/hooks/data_source/test_zfs.py b/tests/unit/hooks/data_source/test_zfs.py index 74ca87b3..f1304543 100644 --- a/tests/unit/hooks/data_source/test_zfs.py +++ b/tests/unit/hooks/data_source/test_zfs.py @@ -296,6 +296,9 @@ def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_ def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) dataset = flexmock( name='dataset', mount_point='/mnt/dataset', @@ -338,6 +341,9 @@ def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns(): def test_dump_data_sources_with_no_datasets_skips_snapshots(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) flexmock(module).should_receive('get_datasets_to_backup').and_return(()) flexmock(module.os).should_receive('getpid').and_return(1234) flexmock(module).should_receive('snapshot_dataset').never() @@ -360,6 +366,9 @@ def test_dump_data_sources_with_no_datasets_skips_snapshots(): def test_dump_data_sources_uses_custom_commands(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) dataset = flexmock( name='dataset', mount_point='/mnt/dataset', @@ -409,6 +418,9 @@ def test_dump_data_sources_uses_custom_commands(): def test_dump_data_sources_with_dry_run_skips_commands_and_does_not_touch_patterns(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) flexmock(module).should_receive('get_datasets_to_backup').and_return( (flexmock(name='dataset', mount_point='/mnt/dataset'),) ) @@ -433,6 +445,9 @@ def test_dump_data_sources_with_dry_run_skips_commands_and_does_not_touch_patter def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained_patterns(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) dataset = flexmock( name='dataset', mount_point='/mnt/dataset', diff --git a/tests/unit/hooks/test_command.py b/tests/unit/hooks/test_command.py index 5d3d285a..cd077e39 100644 --- a/tests/unit/hooks/test_command.py +++ b/tests/unit/hooks/test_command.py @@ -48,43 +48,35 @@ def test_make_environment_with_pyinstaller_and_LD_LIBRARY_PATH_ORIG_copies_it_in ) == {'LD_LIBRARY_PATH_ORIG': '/lib/lib/lib', 'LD_LIBRARY_PATH': '/lib/lib/lib'} -def test_execute_hook_invokes_each_command(): +LOGGING_ANSWER = flexmock() + + +def test_execute_hooks_invokes_each_hook_and_command(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module).should_receive('interpolate_context').replace_with( lambda hook_description, command, context: command ) flexmock(module).should_receive('make_environment').and_return({}) - flexmock(module.borgmatic.execute).should_receive('execute_command').with_args( - [':'], - output_log_level=logging.WARNING, - shell=True, - environment={}, - ).once() - module.execute_hook([':'], None, 'config.yaml', 'pre-backup', dry_run=False) + for command in ('foo', 'bar', 'baz'): + flexmock(module.borgmatic.execute).should_receive('execute_command').with_args( + [command], + output_log_level=LOGGING_ANSWER, + shell=True, + environment={}, + ).once() - -def test_execute_hook_with_multiple_commands_invokes_each_command(): - flexmock(module).should_receive('interpolate_context').replace_with( - lambda hook_description, command, context: command + module.execute_hooks( + [{'before': 'create', 'run': ['foo']}, {'before': 'create', 'run': ['bar', 'baz']}], + umask=None, + dry_run=False, ) - flexmock(module).should_receive('make_environment').and_return({}) - flexmock(module.borgmatic.execute).should_receive('execute_command').with_args( - [':'], - output_log_level=logging.WARNING, - shell=True, - environment={}, - ).once() - flexmock(module.borgmatic.execute).should_receive('execute_command').with_args( - ['true'], - output_log_level=logging.WARNING, - shell=True, - environment={}, - ).once() - - module.execute_hook([':', 'true'], None, 'config.yaml', 'pre-backup', dry_run=False) -def test_execute_hook_with_umask_sets_that_umask(): +def test_execute_hooks_with_umask_sets_that_umask(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module).should_receive('interpolate_context').replace_with( lambda hook_description, command, context: command ) @@ -92,42 +84,46 @@ def test_execute_hook_with_umask_sets_that_umask(): flexmock(module.os).should_receive('umask').with_args(0o22).once() flexmock(module).should_receive('make_environment').and_return({}) flexmock(module.borgmatic.execute).should_receive('execute_command').with_args( - [':'], - output_log_level=logging.WARNING, + ['foo'], + output_log_level=logging.ANSWER, shell=True, environment={}, ) - module.execute_hook([':'], 77, 'config.yaml', 'pre-backup', dry_run=False) + module.execute_hooks([{'before': 'create', 'run': ['foo']}], umask=77, dry_run=False) -def test_execute_hook_with_dry_run_skips_commands(): +def test_execute_hooks_with_dry_run_skips_commands(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module).should_receive('interpolate_context').replace_with( lambda hook_description, command, context: command ) flexmock(module).should_receive('make_environment').and_return({}) flexmock(module.borgmatic.execute).should_receive('execute_command').never() - module.execute_hook([':', 'true'], None, 'config.yaml', 'pre-backup', dry_run=True) + module.execute_hooks([{'before': 'create', 'run': ['foo']}], umask=None, dry_run=True) -def test_execute_hook_with_empty_commands_does_not_raise(): - module.execute_hook([], None, 'config.yaml', 'post-backup', dry_run=False) +def test_execute_hooks_with_empty_commands_does_not_raise(): + module.execute_hooks([], umask=None, dry_run=True) -def test_execute_hook_on_error_logs_as_error(): +def test_execute_hooks_with_error_logs_as_error(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module).should_receive('interpolate_context').replace_with( lambda hook_description, command, context: command ) flexmock(module).should_receive('make_environment').and_return({}) flexmock(module.borgmatic.execute).should_receive('execute_command').with_args( - [':'], + ['foo'], output_log_level=logging.ERROR, shell=True, environment={}, ).once() - module.execute_hook([':'], None, 'config.yaml', 'on-error', dry_run=False) + module.execute_hooks([{'after': 'error', 'run': ['foo']}], umask=None, dry_run=False) def test_considered_soft_failure_treats_soft_fail_exit_code_as_soft_fail(): From 45c114973cc41688c7a19c6ce39117fd72f2bc98 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 8 Mar 2025 18:31:16 -0800 Subject: [PATCH 087/226] Add missing test coverage for new/changed code (#1019). --- borgmatic/config/normalize.py | 2 +- borgmatic/hooks/command.py | 5 +- tests/integration/config/test_generate.py | 110 ++++++++ tests/unit/config/test_generate.py | 22 +- tests/unit/config/test_normalize.py | 110 ++++++++ tests/unit/config/test_validate.py | 9 + tests/unit/hooks/test_command.py | 313 ++++++++++++++++++++++ 7 files changed, 555 insertions(+), 16 deletions(-) diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index d8244297..11f21ce0 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -58,7 +58,7 @@ def normalize_sections(config_filename, config): return [] -def make_command_hook_deprecation_log(config_filename, option_name): +def make_command_hook_deprecation_log(config_filename, option_name): # pragma: no cover ''' Given a configuration filename and the name of a configuration option, return a deprecation warning log for it. diff --git a/borgmatic/hooks/command.py b/borgmatic/hooks/command.py index ab4d8af0..4d69782f 100644 --- a/borgmatic/hooks/command.py +++ b/borgmatic/hooks/command.py @@ -88,11 +88,11 @@ def execute_hooks(command_hooks, umask, dry_run, **context): elif 'after' in hook_config: description = f'after {hook_config.get("after")}' else: - raise ValueError('Invalid hook configuration: {hook_config}') + raise ValueError(f'Invalid hook configuration: {hook_config}') if not commands: logger.debug(f'No commands to run for {description} hook') - return + continue commands = [interpolate_context(description, command, context) for command in commands] @@ -134,7 +134,6 @@ class Before_after_hooks: Example use as a context manager: - with borgmatic.hooks.command.Before_after_hooks( command_hooks=config.get('commands'), before_after='do_stuff', diff --git a/tests/integration/config/test_generate.py b/tests/integration/config/test_generate.py index 2db4823a..f58d6002 100644 --- a/tests/integration/config/test_generate.py +++ b/tests/integration/config/test_generate.py @@ -16,6 +16,116 @@ def test_insert_newline_before_comment_does_not_raise(): module.insert_newline_before_comment(config, field_name) +def test_schema_to_sample_configuration_comments_out_non_default_options(): + schema = { + 'type': 'object', + 'properties': dict( + [ + ('field1', {'example': 'Example 1'}), + ('field2', {'example': 'Example 2'}), + ('source_directories', {'example': 'Example 3'}), + ] + ), + } + + config = module.schema_to_sample_configuration(schema) + + assert config == dict( + [ + ('field1', 'Example 1'), + ('field2', 'Example 2'), + ('source_directories', 'Example 3'), + ] + ) + assert 'COMMENT_OUT' in config.ca.items['field1'][1][-1]._value + assert 'COMMENT_OUT' in config.ca.items['field2'][1][-1]._value + assert 'source_directories' not in config.ca.items + + +def test_schema_to_sample_configuration_comments_out_non_source_config_options(): + schema = { + 'type': 'object', + 'properties': dict( + [ + ('field1', {'example': 'Example 1'}), + ('field2', {'example': 'Example 2'}), + ('field3', {'example': 'Example 3'}), + ] + ), + } + source_config = {'field3': 'value'} + + config = module.schema_to_sample_configuration(schema, source_config) + + assert config == dict( + [ + ('field1', 'Example 1'), + ('field2', 'Example 2'), + ('field3', 'Example 3'), + ] + ) + assert 'COMMENT_OUT' in config.ca.items['field1'][1][-1]._value + assert 'COMMENT_OUT' in config.ca.items['field2'][1][-1]._value + assert 'field3' not in config.ca.items + + +def test_schema_to_sample_configuration_comments_out_non_default_options_in_sequence_of_maps(): + schema = { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': dict( + [ + ('field1', {'example': 'Example 1'}), + ('field2', {'example': 'Example 2'}), + ('source_directories', {'example': 'Example 3'}), + ] + ), + }, + } + + config = module.schema_to_sample_configuration(schema) + + assert config == [ + dict( + [('field1', 'Example 1'), ('field2', 'Example 2'), ('source_directories', 'Example 3')] + ) + ] + + # The first field in a sequence does not get commented. + assert 'field1' not in config[0].ca.items + assert 'COMMENT_OUT' in config[0].ca.items['field2'][1][-1]._value + assert 'source_directories' not in config[0].ca.items + + +def test_schema_to_sample_configuration_comments_out_non_source_config_options_in_sequence_of_maps(): + schema = { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': dict( + [ + ('field1', {'example': 'Example 1'}), + ('field2', {'example': 'Example 2'}), + ('field3', {'example': 'Example 3'}), + ] + ), + }, + } + source_config = [{'field3': 'value'}] + + config = module.schema_to_sample_configuration(schema, source_config) + + assert config == [ + dict([('field1', 'Example 1'), ('field2', 'Example 2'), ('field3', 'Example 3')]) + ] + + # The first field in a sequence does not get commented. + assert 'field1' not in config[0].ca.items + assert 'COMMENT_OUT' in config[0].ca.items['field2'][1][-1]._value + assert 'field3' not in config[0].ca.items + + def test_comment_out_line_skips_blank_line(): line = ' \n' diff --git a/tests/unit/config/test_generate.py b/tests/unit/config/test_generate.py index 9b880df6..a10c422d 100644 --- a/tests/unit/config/test_generate.py +++ b/tests/unit/config/test_generate.py @@ -1,5 +1,3 @@ -from collections import OrderedDict - import pytest from flexmock import flexmock @@ -9,7 +7,7 @@ from borgmatic.config import generate as module def test_get_properties_with_simple_object(): schema = { 'type': 'object', - 'properties': OrderedDict( + 'properties': dict( [ ('field1', {'example': 'Example'}), ] @@ -24,7 +22,7 @@ def test_get_properties_merges_one_of_list_properties(): 'type': 'object', 'oneOf': [ { - 'properties': OrderedDict( + 'properties': dict( [ ('field1', {'example': 'Example 1'}), ('field2', {'example': 'Example 2'}), @@ -32,7 +30,7 @@ def test_get_properties_merges_one_of_list_properties(): ), }, { - 'properties': OrderedDict( + 'properties': dict( [ ('field2', {'example': 'Example 2'}), ('field3', {'example': 'Example 3'}), @@ -50,7 +48,7 @@ def test_get_properties_merges_one_of_list_properties(): def test_schema_to_sample_configuration_generates_config_map_with_examples(): schema = { 'type': 'object', - 'properties': OrderedDict( + 'properties': dict( [ ('field1', {'example': 'Example 1'}), ('field2', {'example': 'Example 2'}), @@ -59,12 +57,12 @@ def test_schema_to_sample_configuration_generates_config_map_with_examples(): ), } flexmock(module).should_receive('get_properties').and_return(schema['properties']) - flexmock(module.ruamel.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict) + flexmock(module.ruamel.yaml.comments).should_receive('CommentedMap').replace_with(dict) flexmock(module).should_receive('add_comments_to_configuration_object') config = module.schema_to_sample_configuration(schema) - assert config == OrderedDict( + assert config == dict( [ ('field1', 'Example 1'), ('field2', 'Example 2'), @@ -88,7 +86,7 @@ def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_e 'type': 'array', 'items': { 'type': 'object', - 'properties': OrderedDict( + 'properties': dict( [('field1', {'example': 'Example 1'}), ('field2', {'example': 'Example 2'})] ), }, @@ -100,7 +98,7 @@ def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_e config = module.schema_to_sample_configuration(schema) - assert config == [OrderedDict([('field1', 'Example 1'), ('field2', 'Example 2')])] + assert config == [dict([('field1', 'Example 1'), ('field2', 'Example 2')])] def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_multiple_types(): @@ -108,7 +106,7 @@ def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_m 'type': 'array', 'items': { 'type': ['object', 'null'], - 'properties': OrderedDict( + 'properties': dict( [('field1', {'example': 'Example 1'}), ('field2', {'example': 'Example 2'})] ), }, @@ -120,7 +118,7 @@ def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_m config = module.schema_to_sample_configuration(schema) - assert config == [OrderedDict([('field1', 'Example 1'), ('field2', 'Example 2')])] + assert config == [dict([('field1', 'Example 1'), ('field2', 'Example 2')])] def test_schema_to_sample_configuration_with_unsupported_schema_raises(): diff --git a/tests/unit/config/test_normalize.py b/tests/unit/config/test_normalize.py index edef6695..abd7e54d 100644 --- a/tests/unit/config/test_normalize.py +++ b/tests/unit/config/test_normalize.py @@ -123,6 +123,114 @@ def test_normalize_sections_with_only_scalar_raises(): module.normalize_sections('test.yaml', config) +@pytest.mark.parametrize( + 'config,expected_config,produces_logs', + ( + ( + {'before_actions': ['foo', 'bar'], 'after_actions': ['baz']}, + { + 'commands': [ + {'before': 'repository', 'run': ['foo', 'bar']}, + {'after': 'repository', 'run': ['baz']}, + ] + }, + True, + ), + ( + {'before_backup': ['foo', 'bar'], 'after_backup': ['baz']}, + { + 'commands': [ + {'before': 'action', 'when': ['create'], 'run': ['foo', 'bar']}, + {'after': 'action', 'when': ['create'], 'run': ['baz']}, + ] + }, + True, + ), + ( + {'before_prune': ['foo', 'bar'], 'after_prune': ['baz']}, + { + 'commands': [ + {'before': 'action', 'when': ['prune'], 'run': ['foo', 'bar']}, + {'after': 'action', 'when': ['prune'], 'run': ['baz']}, + ] + }, + True, + ), + ( + {'before_compact': ['foo', 'bar'], 'after_compact': ['baz']}, + { + 'commands': [ + {'before': 'action', 'when': ['compact'], 'run': ['foo', 'bar']}, + {'after': 'action', 'when': ['compact'], 'run': ['baz']}, + ] + }, + True, + ), + ( + {'before_check': ['foo', 'bar'], 'after_check': ['baz']}, + { + 'commands': [ + {'before': 'action', 'when': ['check'], 'run': ['foo', 'bar']}, + {'after': 'action', 'when': ['check'], 'run': ['baz']}, + ] + }, + True, + ), + ( + {'before_extract': ['foo', 'bar'], 'after_extract': ['baz']}, + { + 'commands': [ + {'before': 'action', 'when': ['extract'], 'run': ['foo', 'bar']}, + {'after': 'action', 'when': ['extract'], 'run': ['baz']}, + ] + }, + True, + ), + ( + {'on_error': ['foo', 'bar']}, + { + 'commands': [ + { + 'after': 'error', + 'when': ['create', 'prune', 'compact', 'check'], + 'run': ['foo', 'bar'], + }, + ] + }, + True, + ), + ( + {'before_everything': ['foo', 'bar'], 'after_everything': ['baz']}, + { + 'commands': [ + {'before': 'everything', 'when': ['create'], 'run': ['foo', 'bar']}, + {'after': 'everything', 'when': ['create'], 'run': ['baz']}, + ] + }, + True, + ), + ( + {'other': 'options', 'unrelated_to': 'commands'}, + {'other': 'options', 'unrelated_to': 'commands'}, + False, + ), + ), +) +def test_normalize_commands_moves_individual_command_hooks_to_unified_commands( + config, expected_config, produces_logs +): + flexmock(module).should_receive('make_command_hook_deprecation_log').and_return(flexmock()) + + logs = module.normalize_commands('test.yaml', config) + + assert config == expected_config + + if produces_logs: + assert logs + else: + assert logs == [] + + @pytest.mark.parametrize( 'config,expected_config,produces_logs', ( @@ -262,6 +370,7 @@ def test_normalize_applies_hard_coded_normalization_to_config( config, expected_config, produces_logs ): flexmock(module).should_receive('normalize_sections').and_return([]) + flexmock(module).should_receive('normalize_commands').and_return([]) logs = module.normalize('test.yaml', config) expected_config.setdefault('bootstrap', {}) @@ -276,6 +385,7 @@ def test_normalize_applies_hard_coded_normalization_to_config( def test_normalize_config_with_borgmatic_source_directory_warns(): flexmock(module).should_receive('normalize_sections').and_return([]) + flexmock(module).should_receive('normalize_commands').and_return([]) logs = module.normalize('test.yaml', {'borgmatic_source_directory': '~/.borgmatic'}) diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index 5bc08039..8490bd07 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -202,6 +202,15 @@ def test_guard_configuration_contains_repository_does_not_raise_when_repository_ ) +def test_guard_configuration_contains_repository_does_not_raise_when_repository_is_none(): + flexmock(module).should_receive('repositories_match').never() + + module.guard_configuration_contains_repository( + repository=None, + configurations={'config.yaml': {'repositories': [{'path': 'foo/bar', 'label': 'repo'}]}}, + ) + + def test_guard_configuration_contains_repository_errors_when_repository_does_not_match(): flexmock(module).should_receive('repositories_match').and_return(False) diff --git a/tests/unit/hooks/test_command.py b/tests/unit/hooks/test_command.py index cd077e39..b832d3dd 100644 --- a/tests/unit/hooks/test_command.py +++ b/tests/unit/hooks/test_command.py @@ -1,6 +1,7 @@ import logging import subprocess +import pytest from flexmock import flexmock from borgmatic.hooks import command as module @@ -48,6 +49,245 @@ def test_make_environment_with_pyinstaller_and_LD_LIBRARY_PATH_ORIG_copies_it_in ) == {'LD_LIBRARY_PATH_ORIG': '/lib/lib/lib', 'LD_LIBRARY_PATH': '/lib/lib/lib'} +@pytest.mark.parametrize( + 'hooks,filters,expected_hooks', + ( + ( + ( + { + 'before': 'action', + 'run': ['foo'], + }, + { + 'after': 'action', + 'run': ['bar'], + }, + { + 'before': 'repository', + 'run': ['baz'], + }, + ), + {}, + ( + { + 'before': 'action', + 'run': ['foo'], + }, + { + 'after': 'action', + 'run': ['bar'], + }, + { + 'before': 'repository', + 'run': ['baz'], + }, + ), + ), + ( + ( + { + 'before': 'action', + 'run': ['foo'], + }, + { + 'after': 'action', + 'run': ['bar'], + }, + { + 'before': 'repository', + 'run': ['baz'], + }, + ), + { + 'before': 'action', + }, + ( + { + 'before': 'action', + 'run': ['foo'], + }, + ), + ), + ( + ( + { + 'after': 'action', + 'run': ['foo'], + }, + { + 'before': 'action', + 'run': ['bar'], + }, + { + 'after': 'repository', + 'run': ['baz'], + }, + ), + { + 'after': 'action', + }, + ( + { + 'after': 'action', + 'run': ['foo'], + }, + ), + ), + ( + ( + { + 'before': 'dump_data_sources', + 'hooks': ['postgresql'], + 'run': ['foo'], + }, + { + 'before': 'dump_data_sources', + 'hooks': ['lvm'], + 'run': ['bar'], + }, + { + 'after': 'dump_data_sources', + 'hooks': ['lvm'], + 'run': ['baz'], + }, + ), + { + 'before': 'dump_data_sources', + 'hook_name': 'lvm', + }, + ( + { + 'before': 'dump_data_sources', + 'hooks': ['lvm'], + 'run': ['bar'], + }, + ), + ), + ( + ( + { + 'before': 'dump_data_sources', + 'run': ['foo'], + }, + { + 'before': 'dump_data_sources', + 'run': ['bar'], + }, + { + 'after': 'dump_data_sources', + 'run': ['baz'], + }, + ), + { + 'before': 'dump_data_sources', + 'hook_name': 'lvm', + }, + ( + { + 'before': 'dump_data_sources', + 'run': ['foo'], + }, + { + 'before': 'dump_data_sources', + 'run': ['bar'], + }, + ), + ), + ( + ( + { + 'before': 'dump_data_sources', + 'hooks': ['postgresql', 'zfs', 'lvm'], + 'run': ['foo'], + }, + ), + { + 'before': 'dump_data_sources', + 'hook_name': 'lvm', + }, + ( + { + 'before': 'dump_data_sources', + 'hooks': ['postgresql', 'zfs', 'lvm'], + 'run': ['foo'], + }, + ), + ), + ( + ( + { + 'before': 'action', + 'when': ['create'], + 'run': ['foo'], + }, + { + 'before': 'action', + 'when': ['prune'], + 'run': ['bar'], + }, + { + 'before': 'action', + 'when': ['compact'], + 'run': ['baz'], + }, + ), + { + 'before': 'action', + 'action_names': ['create', 'compact', 'extract'], + }, + ( + { + 'before': 'action', + 'when': ['create'], + 'run': ['foo'], + }, + { + 'before': 'action', + 'when': ['compact'], + 'run': ['baz'], + }, + ), + ), + ( + ( + { + 'before': 'action', + 'run': ['foo'], + }, + { + 'before': 'action', + 'run': ['bar'], + }, + { + 'before': 'action', + 'run': ['baz'], + }, + ), + { + 'before': 'action', + 'action_names': ['create', 'compact', 'extract'], + }, + ( + { + 'before': 'action', + 'run': ['foo'], + }, + { + 'before': 'action', + 'run': ['bar'], + }, + { + 'before': 'action', + 'run': ['baz'], + }, + ), + ), + ), +) +def test_filter_hooks(hooks, filters, expected_hooks): + assert module.filter_hooks(hooks, **filters) == expected_hooks + + LOGGING_ANSWER = flexmock() @@ -126,6 +366,79 @@ def test_execute_hooks_with_error_logs_as_error(): module.execute_hooks([{'after': 'error', 'run': ['foo']}], umask=None, dry_run=False) +def test_execute_hooks_with_before_or_after_raises(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER + flexmock(module).should_receive('interpolate_context').never() + flexmock(module).should_receive('make_environment').never() + flexmock(module.borgmatic.execute).should_receive('execute_command').never() + + with pytest.raises(ValueError): + module.execute_hooks( + [ + {'erstwhile': 'create', 'run': ['foo']}, + {'erstwhile': 'create', 'run': ['bar', 'baz']}, + ], + umask=None, + dry_run=False, + ) + + +def test_execute_hooks_without_commands_to_run_does_not_raise(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER + flexmock(module).should_receive('interpolate_context').replace_with( + lambda hook_description, command, context: command + ) + flexmock(module).should_receive('make_environment').and_return({}) + + for command in ('foo', 'bar'): + flexmock(module.borgmatic.execute).should_receive('execute_command').with_args( + [command], + output_log_level=LOGGING_ANSWER, + shell=True, + environment={}, + ).once() + + module.execute_hooks( + [{'before': 'create', 'run': []}, {'before': 'create', 'run': ['foo', 'bar']}], + umask=None, + dry_run=False, + ) + + +def test_before_after_hooks_calls_command_hooks(): + commands = [ + {'before': 'repository', 'run': ['foo', 'bar']}, + {'after': 'repository', 'run': ['baz']}, + ] + flexmock(module).should_receive('filter_hooks').with_args( + commands, + before='action', + hook_name='myhook', + action_names=['create'], + ).and_return(flexmock()).once() + flexmock(module).should_receive('filter_hooks').with_args( + commands, + after='action', + hook_name='myhook', + action_names=['create'], + ).and_return(flexmock()).once() + flexmock(module).should_receive('execute_hooks').twice() + + with module.Before_after_hooks( + command_hooks=commands, + before_after='action', + umask=1234, + dry_run=False, + hook_name='myhook', + action_names=['create'], + context1='stuff', + context2='such', + ): + pass + + def test_considered_soft_failure_treats_soft_fail_exit_code_as_soft_fail(): error = subprocess.CalledProcessError(module.SOFT_FAIL_EXIT_CODE, 'try again') From 5ab766b51c1ad456556b88a2bed3661e03a65fcf Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 8 Mar 2025 20:55:13 -0800 Subject: [PATCH 088/226] Add a few more missing tests (#1019). --- borgmatic/commands/borgmatic.py | 4 +-- tests/integration/config/test_generate.py | 39 ++++++++++++++++++++--- tests/unit/config/test_generate.py | 37 ++++++++++++++++++++- 3 files changed, 73 insertions(+), 7 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index ddd3fcb4..b16fc665 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -830,7 +830,7 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): configuration_filename=config_filename, ) except (CalledProcessError, ValueError, OSError) as error: - yield from log_error_records('Error running pre-everything hook', error) + yield from log_error_records('Error running before everything hook', error) return # Execute the actions corresponding to each configuration file. @@ -882,7 +882,7 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): configuration_filename=config_filename, ) except (CalledProcessError, ValueError, OSError) as error: - yield from log_error_records('Error running post-everything hook', error) + yield from log_error_records('Error running after everything hook', error) def exit_with_help_link(): # pragma: no cover diff --git a/tests/integration/config/test_generate.py b/tests/integration/config/test_generate.py index f58d6002..42622403 100644 --- a/tests/integration/config/test_generate.py +++ b/tests/integration/config/test_generate.py @@ -262,7 +262,7 @@ def test_add_comments_to_configuration_sequence_of_maps_without_description_does module.add_comments_to_configuration_sequence(config, schema) -def test_add_comments_to_configuration_object_does_not_raise(): +def test_add_comments_to_configuration_comments_out_non_default_options(): # Ensure that it can deal with fields both in the schema and missing from the schema. config = module.ruamel.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)]) schema = { @@ -272,13 +272,44 @@ def test_add_comments_to_configuration_object_does_not_raise(): module.add_comments_to_configuration_object(config, schema) + assert 'COMMENT_OUT' in config.ca.items['foo'][1][-1]._value + assert 'COMMENT_OUT' in config.ca.items['bar'][1][-1]._value + assert 'baz' not in config.ca.items -def test_add_comments_to_configuration_object_with_skip_first_does_not_raise(): - config = module.ruamel.yaml.comments.CommentedMap([('foo', 33)]) - schema = {'type': 'object', 'properties': {'foo': {'description': 'Foo'}}} + +def test_add_comments_to_configuration_comments_out_non_source_config_options(): + # Ensure that it can deal with fields both in the schema and missing from the schema. + config = module.ruamel.yaml.comments.CommentedMap( + [('repositories', 33), ('bar', 44), ('baz', 55)] + ) + schema = { + 'type': 'object', + 'properties': { + 'repositories': {'description': 'repositories'}, + 'bar': {'description': 'Bar'}, + }, + } + + module.add_comments_to_configuration_object(config, schema) + + assert 'repositories' in config.ca.items + assert 'COMMENT_OUT' in config.ca.items['bar'][1][-1]._value + assert 'baz' not in config.ca.items + + +def test_add_comments_to_configuration_object_with_skip_first_does_not_comment_out_first_option(): + config = module.ruamel.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)]) + schema = { + 'type': 'object', + 'properties': {'foo': {'description': 'Foo'}, 'bar': {'description': 'Bar'}}, + } module.add_comments_to_configuration_object(config, schema, skip_first=True) + assert 'foo' not in config.ca.items + assert 'COMMENT_OUT' in config.ca.items['bar'][1][-1]._value + assert 'baz' not in config.ca.items + def test_generate_sample_configuration_does_not_raise(): builtins = flexmock(sys.modules['builtins']) diff --git a/tests/unit/config/test_generate.py b/tests/unit/config/test_generate.py index a10c422d..27dcd01f 100644 --- a/tests/unit/config/test_generate.py +++ b/tests/unit/config/test_generate.py @@ -17,7 +17,7 @@ def test_get_properties_with_simple_object(): assert module.get_properties(schema) == schema['properties'] -def test_get_properties_merges_one_of_list_properties(): +def test_get_properties_merges_oneof_list_properties(): schema = { 'type': 'object', 'oneOf': [ @@ -45,6 +45,41 @@ def test_get_properties_merges_one_of_list_properties(): ) +def test_get_properties_interleaves_oneof_list_properties(): + schema = { + 'type': 'object', + 'oneOf': [ + { + 'properties': dict( + [ + ('field1', {'example': 'Example 1'}), + ('field2', {'example': 'Example 2'}), + ('field3', {'example': 'Example 3'}), + ] + ), + }, + { + 'properties': dict( + [ + ('field4', {'example': 'Example 4'}), + ('field5', {'example': 'Example 5'}), + ] + ), + }, + ], + } + + assert module.get_properties(schema) == dict( + [ + ('field1', {'example': 'Example 1'}), + ('field4', {'example': 'Example 4'}), + ('field2', {'example': 'Example 2'}), + ('field5', {'example': 'Example 5'}), + ('field3', {'example': 'Example 3'}), + ] + ) + + def test_schema_to_sample_configuration_generates_config_map_with_examples(): schema = { 'type': 'object', From 86b138e73b445347a60a3438030f8c965008a2f6 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 8 Mar 2025 21:00:58 -0800 Subject: [PATCH 089/226] Clarify command hook documentation. --- .../add-preparation-and-cleanup-steps-to-backups.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index 02e41f43..aa0fbcfb 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -7,12 +7,13 @@ eleventyNavigation: --- ## Preparation and cleanup hooks -If you find yourself performing preparation tasks before your backup runs, or -cleanup work afterwards, borgmatic hooks may be of interest. Hooks are shell -commands that borgmatic executes for you at various points as it runs, and -they're configured in the `hooks` section of your configuration file. But if -you're looking to backup a database, it's probably easier to use the [database -backup +If you find yourself performing preparation tasks before your backup runs or +cleanup work afterwards, borgmatic command hooks may be of interest. These are +custom shell commands you can configure borgmatic to execute at various points +as it runs. + +But if you're looking to backup a database, it's probably easier to use the +[database backup feature](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/) instead. From b52339652f23ca60d4b627fe21f81f5541cae156 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 9 Mar 2025 09:57:13 -0700 Subject: [PATCH 090/226] Initial command hooks documentation work (#1019). --- borgmatic/config/schema.yaml | 8 +- ...reparation-and-cleanup-steps-to-backups.md | 92 ++++++++++++++++++- 2 files changed, 95 insertions(+), 5 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 24609f40..31129af2 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -961,8 +961,8 @@ properties: - everything - dump_data_sources description: | - Name for the point in borgmatic's execution before - which the commands should be run (required if + Name for the point in borgmatic's execution that + the commands should be run before (required if "after" isn't set): * "action" runs before each action for each repository. @@ -1039,8 +1039,8 @@ properties: - error - dump_data_sources description: | - Name for the point in borgmatic's execution after - which the commands should be run (required if + Name for the point in borgmatic's execution that + the commands should be run after (required if "before" isn't set): * "action" runs after each action for each repository. diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index aa0fbcfb..d6ea1e06 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -17,7 +17,97 @@ But if you're looking to backup a database, it's probably easier to use the feature](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/) instead. -You can specify `before_backup` hooks to perform preparation steps before +New in version 1.9.14 You can +configure command hooks via a list of `commands:` in your borgmatic +configuration file. For example: + +```yaml +commands: + - before: action + when: + - create + run: + - echo "Before create!" + - after: action + when: + - create + - prune + run: + - echo "After create and/or prune!" +``` + +If A `run:` command contains a special YAML character such as a colon, you may +need to quote the entire string (or use a [multiline +string](https://yaml-multiline.info/)) to avoid an error: + +```yaml +commands: + - before: action + when: + - create + run: + - "echo Backup: start" +``` + +Each command has the following options: + + * `before` or `after`: Name for the point in borgmatic's execution that the commands should be run before/after, one of: + * `action` runs before each action for each repository. (Replaces the deprecated `before_create`, `after_prune`, etc.) + * `repository` runs before/after all actions for each repository. (Replaces the deprecated `before_actions`/`after_actions`.) + * `configuration` runs before/after all actions and repositories in the current configuration file. + * `everything` runs before/after all configuration files. (Replaces the deprecated `before_everything`/`after_everything`.) + * `when`: List of actions for which the commands will be run. Defaults to running for all actions. + * `run`: List of one or more shell commands or scripts to run when this command hook is triggered. + +There's also another command hook that works a little differently: + +```yaml +commands: + - before: dump_data_sources + hooks: + - postgresql + run: + - echo "Right before the PostgreSQL database dump!" +``` + +This command hook has the following options: + + * `before` or `after`: `dump_data_sources` + * `hooks`: List of names of other hooks that this command hook applies to. Defaults to all hooks of the relevant type. + * `run`: List of one or more shell commands or scripts to run when this command hook is triggered. + + +### Order of execution + +Here's a way of visualizing how all of these command hooks slot into borgmatic's +execution. + +Let's say you've got a borgmatic configuration file with a configured +repository. And suppose you configure several command hooks and then run +borgmatic for the `create` and `prune` actions. Here's the order of execution: + + * Trigger `before: everything` (from all configuration files). + * Trigger `before: configuration` (from the first configuration file). + * Trigger `before: repository` (for the first repository). + * Trigger `before: action` for `create`. + * Run the `create` action. + * Trigger `after: action` for `create`. + * Trigger `before: action` for `prune`. + * Run the `prune` action. + * Trigger `after: action` for `prune`. + * Trigger `after: repository` (for the first repository). + * Trigger `after: configuration` (from the first configuration file). + * Trigger `after: everything` (from all configuration files). + +You can imagine how this would be extended to multiple repositories and/or +configuration files. + + +### Deprecated command hooks + +Prior to version 1.9.14 The +command hooks worked a little differently. In these older versions of borgmatic, +you can specify `before_backup` hooks to perform preparation steps before running backups and specify `after_backup` hooks to perform cleanup steps afterwards. Here's an example: From 68b6d010714c5fb5a75dd48726c1d8d648d14aa2 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 9 Mar 2025 13:35:22 -0700 Subject: [PATCH 091/226] Fix a regression in which the "exclude_patterns" option didn't expand "~" (#1021). --- NEWS | 2 ++ borgmatic/actions/create.py | 17 ++++++++++++++--- tests/unit/actions/test_create.py | 21 +++++++++++++++------ 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/NEWS b/NEWS index a41ebd1e..87936e40 100644 --- a/NEWS +++ b/NEWS @@ -5,6 +5,8 @@ * #936: Clarify Zabbix monitoring hook documentation about creating items: https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#zabbix-hook * #1017: Fix a regression in which some MariaDB/MySQL passwords were not escaped correctly. + * #1021: Fix a regression in which the "exclude_patterns" option didn't expand "~" (the user's + home directory). This fix means that all "patterns" and "patterns_from" also now expand "~". 1.9.13 * #975: Add a "compression" option to the PostgreSQL database hook. diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index dede448a..0d2bc87a 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -130,8 +130,11 @@ def expand_directory(directory, working_directory): def expand_patterns(patterns, working_directory=None, skip_paths=None): ''' Given a sequence of borgmatic.borg.pattern.Pattern instances and an optional working directory, - expand tildes and globs in each root pattern. Return all the resulting patterns (not just the - root patterns) as a tuple. + expand tildes and globs in each root pattern and expand just tildes in each non-root pattern. + The idea is that non-root patterns may be regular expressions or other pattern styles containing + "*" that borgmatic should not expand as a shell glob. + + Return all the resulting patterns as a tuple. If a set of paths are given to skip, then don't expand any patterns matching them. ''' @@ -153,7 +156,15 @@ def expand_patterns(patterns, working_directory=None, skip_paths=None): ) if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT and pattern.path not in (skip_paths or ()) - else (pattern,) + else ( + borgmatic.borg.pattern.Pattern( + os.path.expanduser(pattern.path), + pattern.type, + pattern.style, + pattern.device, + pattern.source, + ), + ) ) for pattern in patterns ) diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 5e4a4c9f..93dbcb87 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -225,15 +225,24 @@ def test_expand_patterns_considers_none_as_no_patterns(): assert module.expand_patterns(None) == () -def test_expand_patterns_only_considers_root_patterns(): - flexmock(module).should_receive('expand_directory').with_args('~/foo', None).and_return( - ['/root/foo'] +def test_expand_patterns_expands_tildes_and_globs_in_root_patterns(): + flexmock(module.os.path).should_receive('expanduser').never() + flexmock(module).should_receive('expand_directory').and_return( + ['/root/foo/one', '/root/foo/two'] ) - flexmock(module).should_receive('expand_directory').with_args('bar*', None).never() - paths = module.expand_patterns((Pattern('~/foo'), Pattern('bar*', Pattern_type.INCLUDE))) + paths = module.expand_patterns((Pattern('~/foo/*'),)) - assert paths == (Pattern('/root/foo'), Pattern('bar*', Pattern_type.INCLUDE)) + assert paths == (Pattern('/root/foo/one'), Pattern('/root/foo/two')) + + +def test_expand_patterns_expands_only_tildes_in_non_root_patterns(): + flexmock(module).should_receive('expand_directory').never() + flexmock(module.os.path).should_receive('expanduser').and_return('/root/bar/*') + + paths = module.expand_patterns((Pattern('~/bar/*', Pattern_type.INCLUDE),)) + + assert paths == (Pattern('/root/bar/*', Pattern_type.INCLUDE),) def test_device_map_patterns_gives_device_id_per_path(): From 9941d7dc574372bdb6bfb7292cc1ac0a9e6d6f28 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 9 Mar 2025 17:01:46 -0700 Subject: [PATCH 092/226] More docs and command hook context tweaks (#1019). --- borgmatic/commands/borgmatic.py | 4 + ...reparation-and-cleanup-steps-to-backups.md | 95 ++++++------ tests/unit/commands/test_borgmatic.py | 145 +++++++++++++----- 3 files changed, 164 insertions(+), 80 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index b16fc665..eea3793a 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -102,6 +102,8 @@ def run_configuration(config_filename, config, config_paths, arguments): umask=config.get('umask'), dry_run=global_arguments.dry_run, action_names=arguments.keys(), + configuration_filename=config_filename, + log_file=arguments['global'].log_file, ): try: local_borg_version = borg_version.local_borg_version(config, local_path) @@ -828,6 +830,7 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): config.get('umask'), arguments['global'].dry_run, configuration_filename=config_filename, + log_file=arguments['global'].log_file, ) except (CalledProcessError, ValueError, OSError) as error: yield from log_error_records('Error running before everything hook', error) @@ -880,6 +883,7 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): config.get('umask'), arguments['global'].dry_run, configuration_filename=config_filename, + log_file=arguments['global'].log_file, ) except (CalledProcessError, ValueError, OSError) as error: yield from log_error_records('Error running after everything hook', error) diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index d6ea1e06..0d337e76 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -8,24 +8,23 @@ eleventyNavigation: ## Preparation and cleanup hooks If you find yourself performing preparation tasks before your backup runs or -cleanup work afterwards, borgmatic command hooks may be of interest. These are -custom shell commands you can configure borgmatic to execute at various points -as it runs. +doing cleanup work afterwards, borgmatic command hooks may be of interest. These +are custom shell commands you can configure borgmatic to execute at various +points as it runs. -But if you're looking to backup a database, it's probably easier to use the +(But if you're looking to backup a database, it's probably easier to use the [database backup feature](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/) -instead. +instead.) -New in version 1.9.14 You can -configure command hooks via a list of `commands:` in your borgmatic +New in version 1.9.14 Command +hooks are now configured via a list of `commands:` in your borgmatic configuration file. For example: ```yaml commands: - before: action - when: - - create + when: [create] run: - echo "Before create!" - after: action @@ -36,27 +35,27 @@ commands: - echo "After create and/or prune!" ``` -If A `run:` command contains a special YAML character such as a colon, you may +If a `run:` command contains a special YAML character such as a colon, you may need to quote the entire string (or use a [multiline string](https://yaml-multiline.info/)) to avoid an error: ```yaml commands: - before: action - when: - - create + when: [create] run: - - "echo Backup: start" + - "echo Backup: start" ``` -Each command has the following options: +Each command in the `commands:` list has the following options: - * `before` or `after`: Name for the point in borgmatic's execution that the commands should be run before/after, one of: - * `action` runs before each action for each repository. (Replaces the deprecated `before_create`, `after_prune`, etc.) - * `repository` runs before/after all actions for each repository. (Replaces the deprecated `before_actions`/`after_actions`.) - * `configuration` runs before/after all actions and repositories in the current configuration file. - * `everything` runs before/after all configuration files. (Replaces the deprecated `before_everything`/`after_everything`.) - * `when`: List of actions for which the commands will be run. Defaults to running for all actions. + * `before` or `after`: Name for the point in borgmatic's execution that the commands should be run before or after, one of: + * `action` runs before each action for each repository. This replaces the deprecated `before_create`, `after_prune`, etc. + * `repository` runs before or after all actions for each repository. This replaces the deprecated `before_actions` and `after_actions`. + * `configuration` runs before or after all actions and repositories in the current configuration file. + * `everything` runs before or after all configuration files. This replaces the deprecated `before_everything` and `after_everything`. + * `error` runs after an error occurs—and it's only available for `after`. This replaces the deprecated `on_error` hook. + * `when`: Actions (`create`, `prune`, etc.) for which the commands will be run. Defaults to running for all actions. * `run`: List of one or more shell commands or scripts to run when this command hook is triggered. There's also another command hook that works a little differently: @@ -64,8 +63,7 @@ There's also another command hook that works a little differently: ```yaml commands: - before: dump_data_sources - hooks: - - postgresql + hooks: [postgresql] run: - echo "Right before the PostgreSQL database dump!" ``` @@ -73,8 +71,8 @@ commands: This command hook has the following options: * `before` or `after`: `dump_data_sources` - * `hooks`: List of names of other hooks that this command hook applies to. Defaults to all hooks of the relevant type. - * `run`: List of one or more shell commands or scripts to run when this command hook is triggered. + * `hooks`: Names of other hooks that this command hook applies to, e.g. `postgresql`, `mariadb`, `zfs`, `btrfs`, etc. Defaults to all hooks of the relevant type. + * `run`: One or more shell commands or scripts to run when this command hook is triggered. ### Order of execution @@ -86,20 +84,23 @@ Let's say you've got a borgmatic configuration file with a configured repository. And suppose you configure several command hooks and then run borgmatic for the `create` and `prune` actions. Here's the order of execution: - * Trigger `before: everything` (from all configuration files). - * Trigger `before: configuration` (from the first configuration file). - * Trigger `before: repository` (for the first repository). - * Trigger `before: action` for `create`. - * Run the `create` action. - * Trigger `after: action` for `create`. - * Trigger `before: action` for `prune`. - * Run the `prune` action. - * Trigger `after: action` for `prune`. - * Trigger `after: repository` (for the first repository). - * Trigger `after: configuration` (from the first configuration file). - * Trigger `after: everything` (from all configuration files). + * Run `before: everything` hooks (from all configuration files). + * Run `before: configuration` hooks (from the first configuration file). + * Run `before: repository` hooks (for the first repository). + * Run `before: action` hooks for `create`. + * Run `before: dump_data_sources` hooks (e.g. for the PostgreSQL hook). + * Actually dump data sources (e.g. PostgreSQL databases). + * Run `after: dump_data_sources` hooks (e.g. for the PostgreSQL hook). + * Actually run the `create` action. + * Run `after: action` hooks for `create`. + * Run `before: action` hooks for `prune`. + * Actually run the `prune` action. + * Run `after: action` hooks for `prune`. + * Run `after: repository` hooks (for the first repository). + * Run `after: configuration` hooks (from the first configuration file). + * Run `after: everything` hooks (from all configuration files). -You can imagine how this would be extended to multiple repositories and/or +This same order of execution extends to multiple repositories and/or configuration files. @@ -151,18 +152,18 @@ rather than once per repository.) ## Variable interpolation -The before and after action hooks support interpolating particular runtime -variables into the hook command. Here's an example that assumes you provide a -separate shell script: +The command action hooks support interpolating particular runtime variables into +the commands that are run. Here's an example that assumes you provide a separate +shell script: ```yaml -after_prune: - - record-prune.sh "{configuration_filename}" "{repository}" +commands: + - after: action + when: [prune] + run: + - record-prune.sh "{configuration_filename}" "{repository}" ``` -Prior to version 1.8.0 Put -this option in the `hooks:` section of your configuration. - In this example, when the hook is triggered, borgmatic interpolates runtime values into the hook command: the borgmatic configuration filename and the paths of the current Borg repository. Here's the full set of supported @@ -179,6 +180,10 @@ variables you can use here: 1.8.12: label of the current repository as configured in the current borgmatic configuration file +Not all command hooks support all variables. For instance, the `everything` and +`configuration` hooks don't support repository variables because those hooks +don't run in the context of a single repository. + Note that you can also interpolate in [arbitrary environment variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/). diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index ce87cb6b..c2514a72 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -38,7 +38,7 @@ def test_run_configuration_runs_actions_for_each_repository(): expected_results[1:] ) config = {'repositories': [{'path': 'foo'}, {'path': 'bar'}]} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False)} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock())} results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -53,7 +53,7 @@ def test_run_configuration_with_skip_actions_does_not_raise(): flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_return(flexmock()).and_return(flexmock()) config = {'repositories': [{'path': 'foo'}, {'path': 'bar'}], 'skip_actions': ['compact']} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False)} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock())} list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -67,7 +67,10 @@ def test_run_configuration_with_invalid_borg_version_errors(): flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').never() config = {'repositories': [{'path': 'foo'}]} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'prune': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'prune': flexmock(), + } list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -87,7 +90,10 @@ def test_run_configuration_logs_monitor_start_error(): flexmock(module.command).should_receive('filter_hooks') flexmock(module.command).should_receive('execute_hooks') config = {'repositories': [{'path': 'foo'}]} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -105,7 +111,10 @@ def test_run_configuration_bails_for_monitor_start_soft_failure(): flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').never() config = {'repositories': [{'path': 'foo'}, {'path': 'bar'}]} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -125,7 +134,7 @@ def test_run_configuration_logs_actions_error(): flexmock(module.command).should_receive('filter_hooks') flexmock(module.command).should_receive('execute_hooks') config = {'repositories': [{'path': 'foo'}]} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False)} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock())} results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -145,7 +154,10 @@ def test_run_configuration_skips_remaining_actions_for_actions_soft_failure_but_ flexmock(module).should_receive('log_error_records').never() flexmock(module.command).should_receive('considered_soft_failure').and_return(True) config = {'repositories': [{'path': 'foo'}, {'path': 'bar'}]} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -167,7 +179,10 @@ def test_run_configuration_logs_monitor_log_error(): flexmock(module.command).should_receive('filter_hooks') flexmock(module.command).should_receive('execute_hooks') config = {'repositories': [{'path': 'foo'}]} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -188,7 +203,10 @@ def test_run_configuration_still_pings_monitor_for_monitor_log_soft_failure(): flexmock(module).should_receive('run_actions').and_return([]) flexmock(module.command).should_receive('considered_soft_failure').and_return(True) config = {'repositories': [{'path': 'foo'}]} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -210,7 +228,10 @@ def test_run_configuration_logs_monitor_finish_error(): flexmock(module.command).should_receive('filter_hooks') flexmock(module.command).should_receive('execute_hooks') config = {'repositories': [{'path': 'foo'}]} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -231,7 +252,10 @@ def test_run_configuration_bails_for_monitor_finish_soft_failure(): flexmock(module).should_receive('run_actions').and_return([]) flexmock(module.command).should_receive('considered_soft_failure').and_return(True) config = {'repositories': [{'path': 'foo'}]} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -249,7 +273,10 @@ def test_run_configuration_does_not_call_monitoring_hooks_if_monitoring_hooks_ar flexmock(module).should_receive('run_actions').and_return([]) config = {'repositories': [{'path': 'foo'}]} - arguments = {'global': flexmock(monitoring_verbosity=-2, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=-2, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) assert results == [] @@ -268,7 +295,10 @@ def test_run_configuration_logs_on_error_hook_error(): flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError) config = {'repositories': [{'path': 'foo'}]} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -288,7 +318,10 @@ def test_run_configuration_bails_for_on_error_hook_soft_failure(): flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError) config = {'repositories': [{'path': 'foo'}]} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -307,7 +340,10 @@ def test_run_configuration_retries_soft_error(): flexmock(module.command).should_receive('filter_hooks').never() flexmock(module.command).should_receive('execute_hooks').never() config = {'repositories': [{'path': 'foo'}], 'retries': 1} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -336,7 +372,10 @@ def test_run_configuration_retries_hard_error(): flexmock(module.command).should_receive('filter_hooks') flexmock(module.command).should_receive('execute_hooks') config = {'repositories': [{'path': 'foo'}], 'retries': 1} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -360,7 +399,10 @@ def test_run_configuration_retries_repositories_in_order(): flexmock(module.command).should_receive('filter_hooks') flexmock(module.command).should_receive('execute_hooks') config = {'repositories': [{'path': 'foo'}, {'path': 'bar'}]} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -400,7 +442,10 @@ def test_run_configuration_retries_round_robin(): 'repositories': [{'path': 'foo'}, {'path': 'bar'}], 'retries': 1, } - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -438,7 +483,10 @@ def test_run_configuration_with_one_retry(): 'repositories': [{'path': 'foo'}, {'path': 'bar'}], 'retries': 1, } - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -487,7 +535,10 @@ def test_run_configuration_with_retry_wait_does_backoff_after_each_retry(): 'retries': 3, 'retry_wait': 10, } - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -532,7 +583,10 @@ def test_run_configuration_with_multiple_repositories_retries_with_timeout(): 'retries': 1, 'retry_wait': 10, } - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -1441,7 +1495,7 @@ def test_collect_configuration_run_summary_logs_info_for_success(): flexmock(module.command).should_receive('execute_hooks') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return([]) - arguments = {'global': flexmock(dry_run=False)} + arguments = {'global': flexmock(dry_run=False, log_file=flexmock())} logs = tuple( module.collect_configuration_run_summary_logs( @@ -1458,7 +1512,10 @@ def test_collect_configuration_run_summary_executes_hooks_for_create(): flexmock(module.command).should_receive('execute_hooks') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return([]) - arguments = {'create': flexmock(), 'global': flexmock(monitoring_verbosity=1, dry_run=False)} + arguments = { + 'create': flexmock(), + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + } logs = tuple( module.collect_configuration_run_summary_logs( @@ -1475,7 +1532,10 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_extract(): flexmock(module.command).should_receive('execute_hooks') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return([]) - arguments = {'extract': flexmock(repository='repo'), 'global': flexmock(dry_run=False)} + arguments = { + 'extract': flexmock(repository='repo'), + 'global': flexmock(dry_run=False, log_file=flexmock()), + } logs = tuple( module.collect_configuration_run_summary_logs( @@ -1492,7 +1552,7 @@ def test_collect_configuration_run_summary_logs_extract_with_repository_error(): ) expected_logs = (flexmock(),) flexmock(module).should_receive('log_error_records').and_return(expected_logs) - arguments = {'extract': flexmock(repository='repo')} + arguments = {'extract': flexmock(repository='repo', log_file=flexmock())} logs = tuple( module.collect_configuration_run_summary_logs( @@ -1509,7 +1569,10 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_mount(): flexmock(module.command).should_receive('execute_hooks') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return([]) - arguments = {'mount': flexmock(repository='repo'), 'global': flexmock(dry_run=False)} + arguments = { + 'mount': flexmock(repository='repo'), + 'global': flexmock(dry_run=False, log_file=flexmock()), + } logs = tuple( module.collect_configuration_run_summary_logs( @@ -1526,7 +1589,10 @@ def test_collect_configuration_run_summary_logs_mount_with_repository_error(): ) expected_logs = (flexmock(),) flexmock(module).should_receive('log_error_records').and_return(expected_logs) - arguments = {'mount': flexmock(repository='repo'), 'global': flexmock(dry_run=False)} + arguments = { + 'mount': flexmock(repository='repo'), + 'global': flexmock(dry_run=False, log_file=flexmock()), + } logs = tuple( module.collect_configuration_run_summary_logs( @@ -1541,7 +1607,7 @@ def test_collect_configuration_run_summary_logs_missing_configs_error(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') flexmock(module.command).should_receive('filter_hooks') flexmock(module.command).should_receive('execute_hooks') - arguments = {'global': flexmock(config_paths=[])} + arguments = {'global': flexmock(config_paths=[], log_file=flexmock())} expected_logs = (flexmock(),) flexmock(module).should_receive('log_error_records').and_return(expected_logs) @@ -1558,7 +1624,10 @@ def test_collect_configuration_run_summary_logs_pre_hook_error(): flexmock(module.command).should_receive('execute_hooks').and_raise(ValueError) expected_logs = (flexmock(),) flexmock(module).should_receive('log_error_records').and_return(expected_logs) - arguments = {'create': flexmock(), 'global': flexmock(monitoring_verbosity=1, dry_run=False)} + arguments = { + 'create': flexmock(), + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + } logs = tuple( module.collect_configuration_run_summary_logs( @@ -1577,7 +1646,10 @@ def test_collect_configuration_run_summary_logs_post_hook_error(): flexmock(module).should_receive('run_configuration').and_return([]) expected_logs = (flexmock(),) flexmock(module).should_receive('log_error_records').and_return(expected_logs) - arguments = {'create': flexmock(), 'global': flexmock(monitoring_verbosity=1, dry_run=False)} + arguments = { + 'create': flexmock(), + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + } logs = tuple( module.collect_configuration_run_summary_logs( @@ -1596,7 +1668,7 @@ def test_collect_configuration_run_summary_logs_for_list_with_archive_and_reposi flexmock(module).should_receive('log_error_records').and_return(expected_logs) arguments = { 'list': flexmock(repository='repo', archive='test'), - 'global': flexmock(dry_run=False), + 'global': flexmock(dry_run=False, log_file=flexmock()), } logs = tuple( @@ -1616,7 +1688,7 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_list(): flexmock(module).should_receive('run_configuration').and_return([]) arguments = { 'list': flexmock(repository='repo', archive=None), - 'global': flexmock(dry_run=False), + 'global': flexmock(dry_run=False, log_file=flexmock()), } logs = tuple( @@ -1637,7 +1709,7 @@ def test_collect_configuration_run_summary_logs_run_configuration_error(): [logging.makeLogRecord(dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg='Error'))] ) flexmock(module).should_receive('log_error_records').and_return([]) - arguments = {'global': flexmock(dry_run=False)} + arguments = {'global': flexmock(dry_run=False, log_file=flexmock())} logs = tuple( module.collect_configuration_run_summary_logs( @@ -1658,7 +1730,10 @@ def test_collect_configuration_run_summary_logs_run_umount_error(): flexmock(module).should_receive('log_error_records').and_return( [logging.makeLogRecord(dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg='Error'))] ) - arguments = {'umount': flexmock(mount_point='/mnt'), 'global': flexmock(dry_run=False)} + arguments = { + 'umount': flexmock(mount_point='/mnt'), + 'global': flexmock(dry_run=False, log_file=flexmock()), + } logs = tuple( module.collect_configuration_run_summary_logs( @@ -1680,7 +1755,7 @@ def test_collect_configuration_run_summary_logs_outputs_merged_json_results(): stdout = flexmock() stdout.should_receive('write').with_args('["foo", "bar", "baz"]').once() flexmock(module.sys).stdout = stdout - arguments = {'global': flexmock(dry_run=False)} + arguments = {'global': flexmock(dry_run=False, log_file=flexmock())} tuple( module.collect_configuration_run_summary_logs( From 4ee7f726965d1fb9ec05f43aa9e25fab0bfa1e25 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 9 Mar 2025 23:04:55 -0700 Subject: [PATCH 093/226] Fix an error in the Btrfs hook when attempting to snapshot a read-only subvolume (#1023). --- NEWS | 2 + borgmatic/hooks/data_source/btrfs.py | 47 +++++++++++++++- docs/how-to/snapshot-your-filesystems.md | 5 +- tests/unit/hooks/data_source/test_btrfs.py | 62 ++++++++++++++++++++++ 4 files changed, 113 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 87936e40..16e3db14 100644 --- a/NEWS +++ b/NEWS @@ -7,6 +7,8 @@ * #1017: Fix a regression in which some MariaDB/MySQL passwords were not escaped correctly. * #1021: Fix a regression in which the "exclude_patterns" option didn't expand "~" (the user's home directory). This fix means that all "patterns" and "patterns_from" also now expand "~". + * #1023: Fix an error in the Btrfs hook when attempting to snapshot a read-only subvolume. Now, + read-only subvolumes are ignored since Btrfs can't actually snapshot them. 1.9.13 * #975: Add a "compression" option to the PostgreSQL database hook. diff --git a/borgmatic/hooks/data_source/btrfs.py b/borgmatic/hooks/data_source/btrfs.py index ec6d33df..f46cbc2e 100644 --- a/borgmatic/hooks/data_source/btrfs.py +++ b/borgmatic/hooks/data_source/btrfs.py @@ -48,6 +48,47 @@ def get_subvolume_mount_points(findmnt_command): Subvolume = collections.namedtuple('Subvolume', ('path', 'contained_patterns'), defaults=((),)) +def get_subvolume_property(btrfs_command, subvolume_path, property_name): + output = borgmatic.execute.execute_command_and_capture_output( + tuple(btrfs_command.split(' ')) + + ( + 'property', + 'get', + '-t', # Type. + 'subvol', + subvolume_path, + property_name, + ), + ) + + try: + value = output.strip().split('=')[1] + except IndexError: + raise ValueError(f'Invalid {btrfs_command} property output') + + return { + 'true': True, + 'false': False, + }.get(value, value) + + +def omit_read_only_subvolume_mount_points(btrfs_command, subvolume_paths): + ''' + Given a Btrfs command to run and a sequence of Btrfs subvolume mount points, filter them down to + just those that are read-write. The idea is that Btrfs can't actually snapshot a read-only + subvolume, so we should just ignore them. + ''' + retained_subvolume_paths = [] + + for subvolume_path in subvolume_paths: + if get_subvolume_property(btrfs_command, subvolume_path, 'ro'): + logger.debug(f'Ignoring Btrfs subvolume {subvolume_path} because it is read-only') + else: + retained_subvolume_paths.append(subvolume_path) + + return tuple(retained_subvolume_paths) + + def get_subvolumes(btrfs_command, findmnt_command, patterns=None): ''' Given a Btrfs command to run and a sequence of configured patterns, find the intersection @@ -67,7 +108,11 @@ def get_subvolumes(btrfs_command, findmnt_command, patterns=None): # backup. Sort the subvolumes from longest to shortest mount points, so longer mount points get # a whack at the candidate pattern piñata before their parents do. (Patterns are consumed during # this process, so no two subvolumes end up with the same contained patterns.) - for mount_point in reversed(get_subvolume_mount_points(findmnt_command)): + for mount_point in reversed( + omit_read_only_subvolume_mount_points( + btrfs_command, get_subvolume_mount_points(findmnt_command) + ) + ): subvolumes.extend( Subvolume(mount_point, contained_patterns) for contained_patterns in ( diff --git a/docs/how-to/snapshot-your-filesystems.md b/docs/how-to/snapshot-your-filesystems.md index bd569957..8dd8bad5 100644 --- a/docs/how-to/snapshot-your-filesystems.md +++ b/docs/how-to/snapshot-your-filesystems.md @@ -148,8 +148,9 @@ feedback](https://torsion.org/borgmatic/#issues) you have on this feature. #### Subvolume discovery -For any subvolume you'd like backed up, add its path to borgmatic's -`source_directories` option. +For any read-write subvolume you'd like backed up, add its path to borgmatic's +`source_directories` option. Btrfs does not support snapshotting read-only +subvolumes. New in version 1.9.6 Or include the mount point as a root pattern with borgmatic's `patterns` or `patterns_from` diff --git a/tests/unit/hooks/data_source/test_btrfs.py b/tests/unit/hooks/data_source/test_btrfs.py index 648902c3..2ba0ccf8 100644 --- a/tests/unit/hooks/data_source/test_btrfs.py +++ b/tests/unit/hooks/data_source/test_btrfs.py @@ -49,8 +49,61 @@ def test_get_subvolume_mount_points_with_findmnt_json_missing_filesystems_errors module.get_subvolume_mount_points('findmnt') +def test_get_subvolume_property_with_invalid_btrfs_output_errors(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return('invalid') + + with pytest.raises(ValueError): + module.get_subvolume_property('btrfs', '/foo', 'ro') + + +def test_get_subvolume_property_with_true_output_returns_true_bool(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return('ro=true') + + assert module.get_subvolume_property('btrfs', '/foo', 'ro') == True + + +def test_get_subvolume_property_with_false_output_returns_false_bool(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return('ro=false') + + assert module.get_subvolume_property('btrfs', '/foo', 'ro') == False + + +def test_get_subvolume_property_passes_through_general_value(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return('thing=value') + + assert module.get_subvolume_property('btrfs', '/foo', 'thing') == 'value' + + +def test_omit_read_only_subvolume_mount_points_filters_out_read_only(): + flexmock(module).should_receive('get_subvolume_property').with_args( + 'btrfs', '/foo', 'ro' + ).and_return(False) + flexmock(module).should_receive('get_subvolume_property').with_args( + 'btrfs', '/bar', 'ro' + ).and_return(True) + flexmock(module).should_receive('get_subvolume_property').with_args( + 'btrfs', '/baz', 'ro' + ).and_return(False) + + assert module.omit_read_only_subvolume_mount_points('btrfs', ('/foo', '/bar', '/baz')) == ( + '/foo', + '/baz', + ) + + def test_get_subvolumes_collects_subvolumes_matching_patterns(): flexmock(module).should_receive('get_subvolume_mount_points').and_return(('/mnt1', '/mnt2')) + flexmock(module).should_receive('omit_read_only_subvolume_mount_points').and_return( + ('/mnt1', '/mnt2') + ) contained_pattern = Pattern( '/mnt1', @@ -76,6 +129,9 @@ def test_get_subvolumes_collects_subvolumes_matching_patterns(): def test_get_subvolumes_skips_non_root_patterns(): flexmock(module).should_receive('get_subvolume_mount_points').and_return(('/mnt1', '/mnt2')) + flexmock(module).should_receive('omit_read_only_subvolume_mount_points').and_return( + ('/mnt1', '/mnt2') + ) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' @@ -107,6 +163,9 @@ def test_get_subvolumes_skips_non_root_patterns(): def test_get_subvolumes_skips_non_config_patterns(): flexmock(module).should_receive('get_subvolume_mount_points').and_return(('/mnt1', '/mnt2')) + flexmock(module).should_receive('omit_read_only_subvolume_mount_points').and_return( + ('/mnt1', '/mnt2') + ) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' @@ -138,6 +197,9 @@ def test_get_subvolumes_skips_non_config_patterns(): def test_get_subvolumes_without_patterns_collects_all_subvolumes(): flexmock(module).should_receive('get_subvolume_mount_points').and_return(('/mnt1', '/mnt2')) + flexmock(module).should_receive('omit_read_only_subvolume_mount_points').and_return( + ('/mnt1', '/mnt2') + ) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' From bec5a0c0ca6d03350182177ebdd5d11d4d740f43 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Mar 2025 10:15:23 -0700 Subject: [PATCH 094/226] Fix end-to-end tests for Btrfs (#1023). --- tests/end-to-end/commands/fake_btrfs.py | 17 +++++++++++++++-- tests/unit/hooks/data_source/test_btrfs.py | 4 ++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/tests/end-to-end/commands/fake_btrfs.py b/tests/end-to-end/commands/fake_btrfs.py index ee969597..b23fc553 100644 --- a/tests/end-to-end/commands/fake_btrfs.py +++ b/tests/end-to-end/commands/fake_btrfs.py @@ -24,7 +24,14 @@ def parse_arguments(*unparsed_arguments): delete_parser = subvolume_subparser.add_parser('delete') delete_parser.add_argument('snapshot_path') - return global_parser.parse_args(unparsed_arguments) + property_parser = action_parsers.add_parser('property') + property_subparser = property_parser.add_subparsers(dest='subaction') + get_parser = property_subparser.add_parser('get') + get_parser.add_argument('-t', dest='type') + get_parser.add_argument('subvolume_path') + get_parser.add_argument('property_name') + + return (global_parser, global_parser.parse_args(unparsed_arguments)) BUILTIN_SUBVOLUME_LIST_LINES = ( @@ -60,9 +67,13 @@ def print_subvolume_list(arguments, snapshot_paths): def main(): - arguments = parse_arguments(*sys.argv[1:]) + (global_parser, arguments) = parse_arguments(*sys.argv[1:]) snapshot_paths = load_snapshots() + if not hasattr(arguments, 'subaction'): + global_parser.print_help() + sys.exit(1) + if arguments.subaction == 'list': print_subvolume_list(arguments, snapshot_paths) elif arguments.subaction == 'snapshot': @@ -84,6 +95,8 @@ def main(): if snapshot_path.endswith('/' + arguments.snapshot_path) ] save_snapshots(snapshot_paths) + elif arguments.action == 'property' and arguments.subaction == 'get': + print(f'{arguments.property_name}=false') if __name__ == '__main__': diff --git a/tests/unit/hooks/data_source/test_btrfs.py b/tests/unit/hooks/data_source/test_btrfs.py index 2ba0ccf8..fb44b91d 100644 --- a/tests/unit/hooks/data_source/test_btrfs.py +++ b/tests/unit/hooks/data_source/test_btrfs.py @@ -63,7 +63,7 @@ def test_get_subvolume_property_with_true_output_returns_true_bool(): 'execute_command_and_capture_output' ).and_return('ro=true') - assert module.get_subvolume_property('btrfs', '/foo', 'ro') == True + assert module.get_subvolume_property('btrfs', '/foo', 'ro') is True def test_get_subvolume_property_with_false_output_returns_false_bool(): @@ -71,7 +71,7 @@ def test_get_subvolume_property_with_false_output_returns_false_bool(): 'execute_command_and_capture_output' ).and_return('ro=false') - assert module.get_subvolume_property('btrfs', '/foo', 'ro') == False + assert module.get_subvolume_property('btrfs', '/foo', 'ro') is False def test_get_subvolume_property_passes_through_general_value(): From fbdb09b87d5cd3482e117e71a26a6948e80e9caf Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Mar 2025 10:17:36 -0700 Subject: [PATCH 095/226] Bump version for release. --- NEWS | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 16e3db14..6a18c1bd 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -1.9.14.dev0 +1.9.14 * #409: With the PagerDuty monitoring hook, send borgmatic logs to PagerDuty so they show up in the incident UI. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook diff --git a/pyproject.toml b/pyproject.toml index cd1ce7a3..fdded2be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "borgmatic" -version = "1.9.14.dev0" +version = "1.9.14" authors = [ { name="Dan Helfman", email="witten@torsion.org" }, ] From 965740c778c66697ead8ae59dbbb8c0c483bdd43 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Mar 2025 10:37:09 -0700 Subject: [PATCH 096/226] Update version of command hooks since they didn't get released in 1.9.14 (#1019). --- NEWS | 8 +++++--- .../add-preparation-and-cleanup-steps-to-backups.md | 4 ++-- pyproject.toml | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/NEWS b/NEWS index e9b991a6..bff2f68a 100644 --- a/NEWS +++ b/NEWS @@ -1,10 +1,12 @@ +2.0.0dev0 + * #790: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more flexible + "commands:". See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ + 1.9.14 * #409: With the PagerDuty monitoring hook, send borgmatic logs to PagerDuty so they show up in the incident UI. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook - * #790: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more flexible - "commands:". See the documentation for more information: - https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ * #936: Clarify Zabbix monitoring hook documentation about creating items: https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#zabbix-hook * #1017: Fix a regression in which some MariaDB/MySQL passwords were not escaped correctly. diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index 0d337e76..98f20ccf 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -17,7 +17,7 @@ points as it runs. feature](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/) instead.) -New in version 1.9.14 Command +New in version 2.0.0 Command hooks are now configured via a list of `commands:` in your borgmatic configuration file. For example: @@ -106,7 +106,7 @@ configuration files. ### Deprecated command hooks -Prior to version 1.9.14 The +Prior to version 2.0.0 The command hooks worked a little differently. In these older versions of borgmatic, you can specify `before_backup` hooks to perform preparation steps before running backups and specify `after_backup` hooks to perform cleanup steps diff --git a/pyproject.toml b/pyproject.toml index fdded2be..d01b6f8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "borgmatic" -version = "1.9.14" +version = "2.0.0dev0" authors = [ { name="Dan Helfman", email="witten@torsion.org" }, ] From 8817364e6d4083aa570add693142909d6b63c193 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 10 Mar 2025 22:38:48 -0700 Subject: [PATCH 097/226] Documentation on command hooks (#1019). --- NEWS | 2 +- borgmatic/commands/borgmatic.py | 8 +- borgmatic/config/schema.yaml | 9 +- docs/_includes/index.css | 1 + ...reparation-and-cleanup-steps-to-backups.md | 153 ++++++++++----- docs/how-to/monitor-your-backups.md | 181 +++++------------- 6 files changed, 164 insertions(+), 190 deletions(-) diff --git a/NEWS b/NEWS index bff2f68a..a3a2852f 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -2.0.0dev0 +2.0.0.dev0 * #790: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more flexible "commands:". See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index eea3793a..102ceef2 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -85,7 +85,7 @@ def run_configuration(config_filename, config, config_paths, arguments): retries = config.get('retries', 0) retry_wait = config.get('retry_wait', 0) encountered_error = None - error_repository = '' + error_repository = None using_primary_action = {'create', 'prune', 'compact', 'check'}.intersection(arguments) monitoring_log_level = verbosity_to_log_level(global_arguments.monitoring_verbosity) monitoring_hooks_are_activated = using_primary_action and monitoring_log_level != DISABLED @@ -192,7 +192,7 @@ def run_configuration(config_filename, config, config_paths, arguments): error, ) encountered_error = error - error_repository = repository['path'] + error_repository = repository try: if monitoring_hooks_are_activated: @@ -246,7 +246,9 @@ def run_configuration(config_filename, config, config_paths, arguments): config.get('umask'), global_arguments.dry_run, configuration_filename=config_filename, - repository=error_repository, + log_file=arguments['global'].log_file, + repository=error_repository.get('path', '') if error_repository else '', + repository_label=error_repository.get('label', '') if error_repository else '', error=encountered_error, output=getattr(encountered_error, 'output', ''), ) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 31129af2..13ab0686 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1091,10 +1091,11 @@ properties: - key - borg description: | - List of actions for which the commands will be - run. Defaults to running for all actions. Ignored - for "dump_data_sources", which by its nature only - runs for "create". + Only trigger the hook when borgmatic is run with + particular actions listed here. Defaults to + running for all actions. Ignored for + "dump_data_sources", which by its nature only runs + for "create". example: [create, prune, compact, check] run: type: array diff --git a/docs/_includes/index.css b/docs/_includes/index.css index 56841ecb..c798f2e9 100644 --- a/docs/_includes/index.css +++ b/docs/_includes/index.css @@ -165,6 +165,7 @@ ul { } li { padding: .25em 0; + line-height: 1.5; } li ul { list-style-type: disc; diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index 98f20ccf..33a6593a 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -33,6 +33,9 @@ commands: - prune run: - echo "After create and/or prune!" + - after: error + run: + - echo "Something went wrong!" ``` If a `run:` command contains a special YAML character such as a colon, you may @@ -55,9 +58,12 @@ Each command in the `commands:` list has the following options: * `configuration` runs before or after all actions and repositories in the current configuration file. * `everything` runs before or after all configuration files. This replaces the deprecated `before_everything` and `after_everything`. * `error` runs after an error occurs—and it's only available for `after`. This replaces the deprecated `on_error` hook. - * `when`: Actions (`create`, `prune`, etc.) for which the commands will be run. Defaults to running for all actions. + * `when`: Only trigger the hook when borgmatic is run with particular actions (`create`, `prune`, etc.) listed here. Defaults to running for all actions. * `run`: List of one or more shell commands or scripts to run when this command hook is triggered. +Note that borgmatic does not run `error` hooks if an error occurs within an +`everything` hook. + There's also another command hook that works a little differently: ```yaml @@ -91,10 +97,10 @@ borgmatic for the `create` and `prune` actions. Here's the order of execution: * Run `before: dump_data_sources` hooks (e.g. for the PostgreSQL hook). * Actually dump data sources (e.g. PostgreSQL databases). * Run `after: dump_data_sources` hooks (e.g. for the PostgreSQL hook). - * Actually run the `create` action. + * Actually run the `create` action (e.g. `borg create`). * Run `after: action` hooks for `create`. * Run `before: action` hooks for `prune`. - * Actually run the `prune` action. + * Actually run the `prune` action (e.g. `borg prune`). * Run `after: action` hooks for `prune`. * Run `after: repository` hooks (for the first repository). * Run `after: configuration` hooks (from the first configuration file). @@ -149,53 +155,13 @@ but not if an error occurs in a previous hook or in the backups themselves. (Prior to borgmatic 1.6.0, these hooks instead ran once per configuration file rather than once per repository.) - -## Variable interpolation - -The command action hooks support interpolating particular runtime variables into -the commands that are run. Here's an example that assumes you provide a separate -shell script: - -```yaml -commands: - - after: action - when: [prune] - run: - - record-prune.sh "{configuration_filename}" "{repository}" -``` - -In this example, when the hook is triggered, borgmatic interpolates runtime -values into the hook command: the borgmatic configuration filename and the -paths of the current Borg repository. Here's the full set of supported -variables you can use here: - - * `configuration_filename`: borgmatic configuration filename in which the - hook was defined - * `log_file` - New in version 1.7.12: - path of the borgmatic log file, only set when the `--log-file` flag is used - * `repository`: path of the current repository as configured in the current - borgmatic configuration file - * `repository_label` New in version - 1.8.12: label of the current repository as configured in the current - borgmatic configuration file - -Not all command hooks support all variables. For instance, the `everything` and -`configuration` hooks don't support repository variables because those hooks -don't run in the context of a single repository. - -Note that you can also interpolate in [arbitrary environment -variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/). - - -## Global hooks - You can also use `before_everything` and `after_everything` hooks to perform global setup or cleanup: ```yaml before_everything: - set-up-stuff-globally + after_everything: - clean-up-stuff-globally ``` @@ -213,13 +179,102 @@ but only if there is a `create` action. It runs even if an error occurs during a backup or a backup hook, but not if an error occurs during a `before_everything` hook. +`on_error` hooks run when an error occurs, but only if there is a `create`, +`prune`, `compact`, or `check` action. For instance, borgmatic can run +configurable shell commands to fire off custom error notifications or take other +actions, so you can get alerted as soon as something goes wrong. Here's a +not-so-useful example: -## Error hooks +```yaml +on_error: + - echo "Error while creating a backup or running a backup hook." +``` -borgmatic also runs `on_error` hooks if an error occurs, either when creating -a backup or running a backup hook. See the [monitoring and alerting -documentation](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/) -for more information. +Prior to version 1.8.0 Put +this option in the `hooks:` section of your configuration. + +The `on_error` hook supports interpolating particular runtime variables into +the hook command. Here's an example that assumes you provide a separate shell +script to handle the alerting: + +```yaml +on_error: + - send-text-message.sh +``` + +borgmatic does not run `on_error` hooks if an error occurs within a +`before_everything` or `after_everything` hook. + + +## Variable interpolation + +The command action hooks support interpolating particular runtime variables into +the commands that are run. Here's are a couple examples that assume you provide +separate shell scripts: + +```yaml +commands: + - after: action + when: [prune] + run: + - record-prune.sh {configuration_filename} {repository} + - after: error + when: [create] + run: + - send-text-message.sh {configuration_filename} {repository} +``` + +In this example, when the hook is triggered, borgmatic interpolates runtime +values into each hook command: the borgmatic configuration filename and the +paths of the current Borg repository. + +Here's the full set of supported variables you can use here: + + * `configuration_filename`: borgmatic configuration filename in which the + hook was defined + * `log_file` + New in version 1.7.12: + path of the borgmatic log file, only set when the `--log-file` flag is used + * `repository`: path of the current repository as configured in the current + borgmatic configuration file, if applicable to the current hook + * `repository_label` New in version + 1.8.12: label of the current repository as configured in the current + borgmatic configuration file, if applicable to the current hook + * `error`: the error message itself, only applies to `error` hooks + * `output`: output of the command that failed, only applies to `error` hooks + (may be blank if an error occurred without running a command) + +Not all command hooks support all variables. For instance, the `everything` and +`configuration` hooks don't support repository variables because those hooks +don't run in the context of a single repository. But the deprecated command +hooks (`before_backup`, `on_error`, etc.) do generally support variable +interpolation. + +borgmatic automatically escapes these interpolated values to prevent shell +injection attacks. One implication is that you shouldn't wrap the interpolated +values in your own quotes, as that will interfere with the quoting performed by +borgmatic and result in your command receiving incorrect arguments. For +instance, this won't work: + +```yaml +commands: + - after: error + run: + # Don't do this! It won't work, as the {error} value is already quoted. + - send-text-message.sh "Uh oh: {error}" +``` + +Do this instead: + +```yaml +commands: + - after: error + run: + - send-text-message.sh {error} +``` + +Note that you can also interpolate [arbitrary environment +variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/). ## Hook output diff --git a/docs/how-to/monitor-your-backups.md b/docs/how-to/monitor-your-backups.md index 9bac14d6..9cdbd4be 100644 --- a/docs/how-to/monitor-your-backups.md +++ b/docs/how-to/monitor-your-backups.md @@ -14,140 +14,55 @@ and alerting comes in. There are several different ways you can monitor your backups and find out whether they're succeeding. Which of these you choose to do is up to you and -your particular infrastructure. +your particular infrastructure: -### Job runner alerts - -The easiest place to start is with failure alerts from the [scheduled job -runner](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#autopilot) -(cron, systemd, etc.) that's running borgmatic. But note that if the job -doesn't even get scheduled (e.g. due to the job runner not running), you -probably won't get an alert at all! Still, this is a decent first line of -defense, especially when combined with some of the other approaches below. - -### Commands run on error - -The `on_error` hook allows you to run an arbitrary command or script when -borgmatic itself encounters an error running your backups. So for instance, -you can run a script to send yourself a text message alert. But note that if -borgmatic doesn't actually run, this alert won't fire. See [error -hooks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hooks) -below for how to configure this. - -### Third-party monitoring services - -borgmatic integrates with these monitoring services and libraries, pinging -them as backups happen: - - * [Apprise](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#apprise-hook) - * [Cronhub](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook) - * [Cronitor](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook) - * [Grafana Loki](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#loki-hook) - * [Healthchecks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook) - * [ntfy](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#ntfy-hook) - * [PagerDuty](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook) - * [Pushover](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pushover-hook) - * [Sentry](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#sentry-hook) - * [Uptime Kuma](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#uptime-kuma-hook) - * [Zabbix](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#zabbix-hook) - -The idea is that you'll receive an alert when something goes wrong or when the -service doesn't hear from borgmatic for a configured interval (if supported). -See the documentation links above for configuration information. - -While these services and libraries offer different features, you probably only -need to use one of them at most. - - -### Third-party monitoring software - -You can use traditional monitoring software to consume borgmatic JSON output -and track when the last successful backup occurred. See [scripting -borgmatic](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#scripting-borgmatic) -below for how to configure this. - -### Borg hosting providers - -Most [Borg hosting -providers](https://torsion.org/borgmatic/#hosting-providers) include -monitoring and alerting as part of their offering. This gives you a dashboard -to check on all of your backups, and can alert you if the service doesn't hear -from borgmatic for a configured interval. - -### Consistency checks - -While not strictly part of monitoring, if you want confidence that your -backups are not only running but are restorable as well, you can configure -particular [consistency -checks](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#consistency-check-configuration) -or even script full [extract -tests](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/). - - -## Error hooks - -When an error occurs during a `create`, `prune`, `compact`, or `check` action, -borgmatic can run configurable shell commands to fire off custom error -notifications or take other actions, so you can get alerted as soon as -something goes wrong. Here's a not-so-useful example: - -```yaml -on_error: - - echo "Error while creating a backup or running a backup hook." -``` - -Prior to version 1.8.0 Put -this option in the `hooks:` section of your configuration. - -The `on_error` hook supports interpolating particular runtime variables into -the hook command. Here's an example that assumes you provide a separate shell -script to handle the alerting: - -```yaml -on_error: - - send-text-message.sh {configuration_filename} {repository} -``` - -In this example, when the error occurs, borgmatic interpolates runtime values -into the hook command: the borgmatic configuration filename and the path of -the repository. Here's the full set of supported variables you can use here: - - * `configuration_filename`: borgmatic configuration filename in which the - error occurred - * `repository`: path of the repository in which the error occurred (may be - blank if the error occurs in a hook) - * `error`: the error message itself - * `output`: output of the command that failed (may be blank if an error - occurred without running a command) - -Note that borgmatic runs the `on_error` hooks only for `create`, `prune`, -`compact`, or `check` actions/hooks in which an error occurs and not other -actions. borgmatic does not run `on_error` hooks if an error occurs within a -`before_everything` or `after_everything` hook. For more about hooks, see the -[borgmatic hooks -documentation](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/), -especially the security information. - -New in version 1.8.7 borgmatic -automatically escapes these interpolated values to prevent shell injection -attacks. One implication of this change is that you shouldn't wrap the -interpolated values in your own quotes, as that will interfere with the -quoting performed by borgmatic and result in your command receiving incorrect -arguments. For instance, this won't work: - - -```yaml -on_error: - # Don't do this! It won't work, as the {error} value is already quoted. - - send-text-message.sh "Uh oh: {error}" -``` - -Do this instead: - -```yaml -on_error: - - send-text-message.sh {error} -``` + * **Job runner alerts**: The easiest place to start is with failure alerts from + the [scheduled job + runner](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#autopilot) + (cron, systemd, etc.) that's running borgmatic. But note that if the job + doesn't even get scheduled (e.g. due to the job runner not running), you + probably won't get an alert at all! Still, this is a decent first line of + defense, especially when combined with some of the other approaches below. + * **Third-party monitoring services:** borgmatic integrates with these monitoring + services and libraries, pinging them as backups happen. The idea is that + you'll receive an alert when something goes wrong or when the service doesn't + hear from borgmatic for a configured interval (if supported). While these + services and libraries offer different features, you probably only need to + use one of them at most. See these documentation links for configuration + information: + * [Apprise](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#apprise-hook) + * [Cronhub](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook) + * [Cronitor](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook) + * [Grafana Loki](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#loki-hook) + * [Healthchecks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook) + * [ntfy](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#ntfy-hook) + * [PagerDuty](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook) + * [Pushover](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pushover-hook) + * [Sentry](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#sentry-hook) + * [Uptime Kuma](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#uptime-kuma-hook) + * [Zabbix](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#zabbix-hook) + * **Third-party monitoring software:** You can use traditional monitoring + software to consume borgmatic JSON output and track when the last successful + backup occurred. See [scripting + borgmatic](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#scripting-borgmatic) + below for how to configure this. + * **Borg hosting providers:** Some [Borg hosting + providers](https://torsion.org/borgmatic/#hosting-providers) include + monitoring and alerting as part of their offering. This gives you a dashboard + to check on all of your backups, and can alert you if the service doesn't + hear from borgmatic for a configured interval. + * **Consistency checks:** While not strictly part of monitoring, if you want + confidence that your backups are not only running but are restorable as well, + you can configure particular [consistency + checks](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#consistency-check-configuration) + or even script full [extract + tests](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/). + * **Commands run on error:** borgmatic's command hooks support running + arbitrary commands or scripts when borgmatic itself encounters an error + running your backups. So for instance, you can run a script to send yourself + a text message alert. But note that if borgmatic doesn't actually run, this + alert won't fire. See the [documentation on command hooks](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/) + for details. ## Healthchecks hook From 7965eb9de3d5ab267201813b4eb80e9245257918 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 11 Mar 2025 11:36:28 -0700 Subject: [PATCH 098/226] Correctly handle errors in command hooks (#1019). --- borgmatic/commands/borgmatic.py | 101 ++++++++++--------- borgmatic/hooks/command.py | 61 +++++++----- tests/unit/commands/test_borgmatic.py | 81 +++++++++------- tests/unit/hooks/test_command.py | 134 ++++++++++++++++++++++++++ 4 files changed, 275 insertions(+), 102 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 102ceef2..7ed854f8 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -207,9 +207,8 @@ def run_configuration(config_filename, config, config_paths, arguments): global_arguments.dry_run, ) except (OSError, CalledProcessError) as error: - if not command.considered_soft_failure(error): - encountered_error = error - yield from log_error_records('Error pinging monitor', error) + encountered_error = error + yield from log_error_records('Error pinging monitor', error) if not encountered_error: try: @@ -231,27 +230,35 @@ def run_configuration(config_filename, config, config_paths, arguments): global_arguments.dry_run, ) except (OSError, CalledProcessError) as error: - if command.considered_soft_failure(error): - return - encountered_error = error yield from log_error_records(f'{config_filename}: Error pinging monitor', error) + else: + return - if encountered_error: - try: - command.execute_hooks( - command.filter_hooks( - config.get('commands'), after='error', action_names=arguments.keys() - ), - config.get('umask'), - global_arguments.dry_run, - configuration_filename=config_filename, - log_file=arguments['global'].log_file, - repository=error_repository.get('path', '') if error_repository else '', - repository_label=error_repository.get('label', '') if error_repository else '', - error=encountered_error, - output=getattr(encountered_error, 'output', ''), - ) + try: + command.execute_hooks( + command.filter_hooks( + config.get('commands'), after='error', action_names=arguments.keys() + ), + config.get('umask'), + global_arguments.dry_run, + configuration_filename=config_filename, + log_file=arguments['global'].log_file, + repository=error_repository.get('path', '') if error_repository else '', + repository_label=error_repository.get('label', '') if error_repository else '', + error=encountered_error, + output=getattr(encountered_error, 'output', ''), + ) + except (OSError, CalledProcessError) as error: + if command.considered_soft_failure(error): + return + + yield from log_error_records( + f'{config_filename}: Error running after error hook', error + ) + + try: + if monitoring_hooks_are_activated: dispatch.call_hooks( 'ping_monitor', config, @@ -268,13 +275,8 @@ def run_configuration(config_filename, config, config_paths, arguments): monitoring_log_level, global_arguments.dry_run, ) - except (OSError, CalledProcessError) as error: - if command.considered_soft_failure(error): - return - - yield from log_error_records( - f'{config_filename}: Error running after error hook', error - ) + except (OSError, CalledProcessError) as error: + yield from log_error_records(f'{config_filename}: Error pinging monitor', error) def run_actions( @@ -843,24 +845,33 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): for config_filename, config in configs.items(): with Log_prefix(config_filename): - results = list(run_configuration(config_filename, config, config_paths, arguments)) - error_logs = tuple( - result for result in results if isinstance(result, logging.LogRecord) - ) - - if error_logs: - yield from log_error_records('An error occurred') - yield from error_logs - else: - yield logging.makeLogRecord( - dict( - levelno=logging.INFO, - levelname='INFO', - msg='Successfully ran configuration file', - ) + try: + results = list(run_configuration(config_filename, config, config_paths, arguments)) + except (OSError, CalledProcessError, ValueError) as error: + yield from log_error_records( + 'Error running configuration file', + error, + levelno=logging.CRITICAL, + log_command_error_output=True, ) - if results: - json_results.extend(results) + else: + error_logs = tuple( + result for result in results if isinstance(result, logging.LogRecord) + ) + + if error_logs: + yield from log_error_records('An error occurred') + yield from error_logs + else: + yield logging.makeLogRecord( + dict( + levelno=logging.INFO, + levelname='INFO', + msg='Successfully ran configuration file', + ) + ) + if results: + json_results.extend(results) if 'umount' in arguments: logger.info(f"Unmounting mount point {arguments['umount'].mount_point}") diff --git a/borgmatic/hooks/command.py b/borgmatic/hooks/command.py index 4d69782f..5403f574 100644 --- a/borgmatic/hooks/command.py +++ b/borgmatic/hooks/command.py @@ -2,6 +2,7 @@ import logging import os import re import shlex +import subprocess import sys import borgmatic.execute @@ -53,7 +54,7 @@ def filter_hooks(command_hooks, before=None, after=None, hook_name=None, action_ ''' return tuple( hook_config - for hook_config in command_hooks + for hook_config in command_hooks or () for config_hook_names in (hook_config.get('hooks'),) for config_action_names in (hook_config.get('when'),) if before is None or hook_config.get('before') == before @@ -97,7 +98,7 @@ def execute_hooks(command_hooks, umask, dry_run, **context): commands = [interpolate_context(description, command, context) for command in commands] if len(commands) == 1: - logger.info(f'Running command for {description} hook{dry_run_label}') + logger.info(f'Running {description} command hook{dry_run_label}') else: logger.info( f'Running {len(commands)} commands for {description} hook{dry_run_label}', @@ -176,33 +177,45 @@ class Before_after_hooks: ''' Run the configured "before" command hooks that match the initialized data points. ''' - execute_hooks( - borgmatic.hooks.command.filter_hooks( - self.command_hooks, - before=self.before_after, - hook_name=self.hook_name, - action_names=self.action_names, - ), - self.umask, - self.dry_run, - **self.context, - ) + try: + execute_hooks( + borgmatic.hooks.command.filter_hooks( + self.command_hooks, + before=self.before_after, + hook_name=self.hook_name, + action_names=self.action_names, + ), + self.umask, + self.dry_run, + **self.context, + ) + except (OSError, subprocess.CalledProcessError) as error: + if considered_soft_failure(error): + return + + raise ValueError(f'Error running before {self.before_after} hook: {error}') def __exit__(self, exception, value, traceback): ''' Run the configured "after" command hooks that match the initialized data points. ''' - execute_hooks( - borgmatic.hooks.command.filter_hooks( - self.command_hooks, - after=self.before_after, - hook_name=self.hook_name, - action_names=self.action_names, - ), - self.umask, - self.dry_run, - **self.context, - ) + try: + execute_hooks( + borgmatic.hooks.command.filter_hooks( + self.command_hooks, + after=self.before_after, + hook_name=self.hook_name, + action_names=self.action_names, + ), + self.umask, + self.dry_run, + **self.context, + ) + except (OSError, subprocess.CalledProcessError) as error: + if considered_soft_failure(error): + return + + raise ValueError(f'Error running before {self.before_after} hook: {error}') def considered_soft_failure(error): diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index c2514a72..e85a6792 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -189,30 +189,6 @@ def test_run_configuration_logs_monitor_log_error(): assert results == expected_results -def test_run_configuration_still_pings_monitor_for_monitor_log_soft_failure(): - flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) - flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) - flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') - flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return( - None - ).and_raise(error).and_return(None).and_return(None).times(5) - flexmock(module).should_receive('log_error_records').never() - flexmock(module).should_receive('Log_prefix').and_return(flexmock()) - flexmock(module).should_receive('run_actions').and_return([]) - flexmock(module.command).should_receive('considered_soft_failure').and_return(True) - config = {'repositories': [{'path': 'foo'}]} - arguments = { - 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), - 'create': flexmock(), - } - - results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) - - assert results == [] - - def test_run_configuration_logs_monitor_finish_error(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) @@ -238,19 +214,38 @@ def test_run_configuration_logs_monitor_finish_error(): assert results == expected_results -def test_run_configuration_bails_for_monitor_finish_soft_failure(): +def test_run_configuration_logs_monitor_fail_error(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') - flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return( - None - ).and_raise(None).and_raise(error) - flexmock(module).should_receive('log_error_records').never() + flexmock(module.dispatch).should_receive('call_hooks') + + # Trigger an error in the monitor finish so that the monitor fail also gets triggered. + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + module.dispatch.Hook_type.MONITORING, + object, + module.monitor.State.FINISH, + object, + object, + ).and_raise(OSError) + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + module.dispatch.Hook_type.MONITORING, + object, + module.monitor.State.FAIL, + object, + object, + ).and_raise(OSError).once() + expected_results = [flexmock()] + flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_return([]) - flexmock(module.command).should_receive('considered_soft_failure').and_return(True) + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') config = {'repositories': [{'path': 'foo'}]} arguments = { 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), @@ -259,7 +254,7 @@ def test_run_configuration_bails_for_monitor_finish_soft_failure(): results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) - assert results == [] + assert results == expected_results + expected_results def test_run_configuration_does_not_call_monitoring_hooks_if_monitoring_hooks_are_disabled(): @@ -1700,7 +1695,7 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_list(): assert {log.levelno for log in logs} == {logging.INFO} -def test_collect_configuration_run_summary_logs_run_configuration_error(): +def test_collect_configuration_run_summary_logs_run_configuration_error_logs(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') flexmock(module.command).should_receive('filter_hooks') flexmock(module.command).should_receive('execute_hooks') @@ -1720,6 +1715,26 @@ def test_collect_configuration_run_summary_logs_run_configuration_error(): assert {log.levelno for log in logs} == {logging.CRITICAL} +def test_collect_configuration_run_summary_logs_run_configuration_exception(): + flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) + flexmock(module).should_receive('run_configuration').and_raise(ValueError) + flexmock(module).should_receive('log_error_records').and_return( + [flexmock(levelno=logging.CRITICAL)] + ) + arguments = {'global': flexmock(dry_run=False, log_file=flexmock())} + + logs = tuple( + module.collect_configuration_run_summary_logs( + {'test.yaml': {}}, config_paths=['/tmp/test.yaml'], arguments=arguments + ) + ) + + assert {log.levelno for log in logs} == {logging.CRITICAL} + + def test_collect_configuration_run_summary_logs_run_umount_error(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') flexmock(module.command).should_receive('filter_hooks') diff --git a/tests/unit/hooks/test_command.py b/tests/unit/hooks/test_command.py index b832d3dd..d111418b 100644 --- a/tests/unit/hooks/test_command.py +++ b/tests/unit/hooks/test_command.py @@ -439,6 +439,140 @@ def test_before_after_hooks_calls_command_hooks(): pass +def test_before_after_hooks_with_before_error_raises_and_skips_after_hook(): + commands = [ + {'before': 'repository', 'run': ['foo', 'bar']}, + {'after': 'repository', 'run': ['baz']}, + ] + flexmock(module).should_receive('filter_hooks').with_args( + commands, + before='action', + hook_name='myhook', + action_names=['create'], + ).and_return(flexmock()).once() + flexmock(module).should_receive('filter_hooks').with_args( + commands, + after='action', + hook_name='myhook', + action_names=['create'], + ).never() + flexmock(module).should_receive('execute_hooks').and_raise(OSError) + flexmock(module).should_receive('considered_soft_failure').and_return(False) + + with pytest.raises(ValueError): + with module.Before_after_hooks( + command_hooks=commands, + before_after='action', + umask=1234, + dry_run=False, + hook_name='myhook', + action_names=['create'], + context1='stuff', + context2='such', + ): + assert False # This should never get called. + + +def test_before_after_hooks_with_before_soft_failure_does_not_raise(): + commands = [ + {'before': 'repository', 'run': ['foo', 'bar']}, + {'after': 'repository', 'run': ['baz']}, + ] + flexmock(module).should_receive('filter_hooks').with_args( + commands, + before='action', + hook_name='myhook', + action_names=['create'], + ).and_return(flexmock()).once() + flexmock(module).should_receive('filter_hooks').with_args( + commands, + after='action', + hook_name='myhook', + action_names=['create'], + ).and_return(flexmock()).once() + flexmock(module).should_receive('execute_hooks').and_raise(OSError) + flexmock(module).should_receive('considered_soft_failure').and_return(True) + + with module.Before_after_hooks( + command_hooks=commands, + before_after='action', + umask=1234, + dry_run=False, + hook_name='myhook', + action_names=['create'], + context1='stuff', + context2='such', + ): + pass + + +def test_before_after_hooks_with_after_error_raises(): + commands = [ + {'before': 'repository', 'run': ['foo', 'bar']}, + {'after': 'repository', 'run': ['baz']}, + ] + flexmock(module).should_receive('filter_hooks').with_args( + commands, + before='action', + hook_name='myhook', + action_names=['create'], + ).and_return(flexmock()).once() + flexmock(module).should_receive('filter_hooks').with_args( + commands, + after='action', + hook_name='myhook', + action_names=['create'], + ).and_return(flexmock()).once() + flexmock(module).should_receive('execute_hooks').and_return(None).and_raise(OSError) + flexmock(module).should_receive('considered_soft_failure').and_return(False) + + with pytest.raises(ValueError): + with module.Before_after_hooks( + command_hooks=commands, + before_after='action', + umask=1234, + dry_run=False, + hook_name='myhook', + action_names=['create'], + context1='stuff', + context2='such', + ): + pass + + +def test_before_after_hooks_with_after_soft_failure_does_not_raise(): + commands = [ + {'before': 'repository', 'run': ['foo', 'bar']}, + {'after': 'repository', 'run': ['baz']}, + ] + flexmock(module).should_receive('filter_hooks').with_args( + commands, + before='action', + hook_name='myhook', + action_names=['create'], + ).and_return(flexmock()).once() + flexmock(module).should_receive('filter_hooks').with_args( + commands, + after='action', + hook_name='myhook', + action_names=['create'], + ).and_return(flexmock()).once() + flexmock(module).should_receive('execute_hooks').and_return(None).and_raise(OSError) + flexmock(module).should_receive('considered_soft_failure').and_return(True) + + with module.Before_after_hooks( + command_hooks=commands, + before_after='action', + umask=1234, + dry_run=False, + hook_name='myhook', + action_names=['create'], + context1='stuff', + context2='such', + ): + pass + + def test_considered_soft_failure_treats_soft_fail_exit_code_as_soft_fail(): error = subprocess.CalledProcessError(module.SOFT_FAIL_EXIT_CODE, 'try again') From b4d24798bf189bec1585234250ea659220cc11a3 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 11 Mar 2025 13:03:58 -0700 Subject: [PATCH 099/226] More command hook documentation updates (#1019). --- ...movable-drive-or-an-intermittent-server.md | 93 ++++++++++--------- 1 file changed, 47 insertions(+), 46 deletions(-) diff --git a/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md b/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md index d54d18f5..b6b65a98 100644 --- a/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md +++ b/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md @@ -29,17 +29,14 @@ concept of "soft failure" come in. This feature leverages [borgmatic command hooks](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/), -so first familiarize yourself with them. The idea is that you write a simple -test in the form of a borgmatic hook to see if backups should proceed or not. +so familiarize yourself with them first. The idea is that you write a simple +test in the form of a borgmatic command hook to see if backups should proceed or +not. The way the test works is that if any of your hook commands return a special exit status of 75, that indicates to borgmatic that it's a temporary failure, and borgmatic should skip all subsequent actions for the current repository. -Prior to version 1.9.0 Soft -failures skipped subsequent actions for *all* repositories in the -configuration file, rather than just for the current repository. - If you return any status besides 75, then it's a standard success or error. (Zero is success; anything else other than 75 is an error). @@ -62,33 +59,37 @@ these options in the `location:` section of your configuration. Prior to version 1.7.10 Omit the `path:` portion of the `repositories` list. -Then, write a `before_backup` hook in that same configuration file that uses -the external `findmnt` utility to see whether the drive is mounted before -proceeding. +Then, make a command hook in that same configuration file that uses the external +`findmnt` utility to see whether the drive is mounted before proceeding. ```yaml -before_backup: +commands: + - before: repository + run: + - findmnt /mnt/removable > /dev/null || exit 75 +``` + +Prior to version 2.0.0 Use the +deprecated `before_actions` hook instead: + +```yaml +before_actions: - findmnt /mnt/removable > /dev/null || exit 75 ``` Prior to version 1.8.0 Put this option in the `hooks:` section of your configuration. +Prior to version 1.7.0 Use +`before_create` or similar instead of `before_actions`, which was introduced in +borgmatic 1.7.0. + What this does is check if the `findmnt` command errors when probing for a particular mount point. If it does error, then it returns exit code 75 to borgmatic. borgmatic logs the soft failure, skips all further actions for the current repository, and proceeds onward to any other repositories and/or configuration files you may have. -If you'd prefer not to use a separate configuration file, and you'd rather -have multiple repositories in a single configuration file, you can make your -`before_backup` soft failure test [vary by -repository](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/#variable-interpolation). -That might require calling out to a separate script though. - -Note that `before_backup` only runs on the `create` action. See below about -optionally using `before_actions` instead. - You can imagine a similar check for the sometimes-online server case: ```yaml @@ -98,50 +99,50 @@ source_directories: repositories: - path: ssh://me@buddys-server.org/./backup.borg -before_backup: - - ping -q -c 1 buddys-server.org > /dev/null || exit 75 +commands: + - before: repository + run: + - ping -q -c 1 buddys-server.org > /dev/null || exit 75 ``` Or to only run backups if the battery level is high enough: ```yaml -before_backup: - - is_battery_percent_at_least.sh 25 +commands: + - before: repository + run: + - is_battery_percent_at_least.sh 25 ``` -(Writing the battery script is left as an exercise to the reader.) - -New in version 1.7.0 The -`before_actions` and `after_actions` hooks run before/after all the actions -(like `create`, `prune`, etc.) for each repository. So if you'd like your soft -failure command hook to run regardless of action, consider using -`before_actions` instead of `before_backup`. +Writing the battery script is left as an exercise to the reader. ## Caveats and details There are some caveats you should be aware of with this feature. - * You'll generally want to put a soft failure command in the `before_backup` + * You'll generally want to put a soft failure command in a `before` command hook, so as to gate whether the backup action occurs. While a soft failure is - also supported in the `after_backup` hook, returning a soft failure there + also supported in an `after` command hook, returning a soft failure there won't prevent any actions from occurring, because they've already occurred! - Similarly, you can return a soft failure from an `on_error` hook, but at + Similarly, you can return a soft failure from an `error` command hook, but at that point it's too late to prevent the error. * Returning a soft failure does prevent further commands in the same hook from - executing. So, like a standard error, it is an "early out". Unlike a standard + executing. So, like a standard error, it is an "early out." Unlike a standard error, borgmatic does not display it in angry red text or consider it a failure. - * Any given soft failure only applies to the a single borgmatic repository - (as of borgmatic 1.9.0). So if you have other repositories you don't want - soft-failed, then make your soft fail test [vary by - repository](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/#variable-interpolation)—or - put anything that you don't want soft-failed (like always-online cloud - backups) in separate configuration files from your soft-failing - repositories. + * New in version 1.9.0 Soft + failures in `action` or `before_*` command hooks only skip the current + repository rather than all repositories in a configuration file. + * If you're writing a soft failure script that you want to vary based on the + current repository, for instance so you can have multiple repositories in a + single configuration file, have a look at [command hook variable + interpolation](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/#variable-interpolation). + And there's always still the option of puting anything that you don't want + soft-failed (like always-online cloud backups) in separate configuration + files from your soft-failing repositories. * The soft failure doesn't have to test anything related to a repository. You - can even perform a test to make sure that individual source directories are - mounted and available. Use your imagination! - * The soft failure feature also works for before/after hooks for other - actions as well. But it is not implemented for `before_everything` or - `after_everything`. + can even perform a test that individual source directories are mounted and + available. Use your imagination! + * Soft failures are not currently implemented for `everything`, + `before_everything`, or `after_everything` command hooks. From 325f53c286c66c3e4da8af2e37dcd5046ebbc1f8 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 11 Mar 2025 14:07:06 -0700 Subject: [PATCH 100/226] Context tweaks + mention configuration upgrade in command hook documentation (#1019). --- borgmatic/commands/borgmatic.py | 8 ++++---- .../add-preparation-and-cleanup-steps-to-backups.md | 13 +++++++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 7ed854f8..b7c6ad52 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -103,7 +103,7 @@ def run_configuration(config_filename, config, config_paths, arguments): dry_run=global_arguments.dry_run, action_names=arguments.keys(), configuration_filename=config_filename, - log_file=arguments['global'].log_file, + log_file=arguments['global'].log_file or '', ): try: local_borg_version = borg_version.local_borg_version(config, local_path) @@ -243,7 +243,7 @@ def run_configuration(config_filename, config, config_paths, arguments): config.get('umask'), global_arguments.dry_run, configuration_filename=config_filename, - log_file=arguments['global'].log_file, + log_file=arguments['global'].log_file or '', repository=error_repository.get('path', '') if error_repository else '', repository_label=error_repository.get('label', '') if error_repository else '', error=encountered_error, @@ -834,7 +834,7 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): config.get('umask'), arguments['global'].dry_run, configuration_filename=config_filename, - log_file=arguments['global'].log_file, + log_file=arguments['global'].log_file or '', ) except (CalledProcessError, ValueError, OSError) as error: yield from log_error_records('Error running before everything hook', error) @@ -896,7 +896,7 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): config.get('umask'), arguments['global'].dry_run, configuration_filename=config_filename, - log_file=arguments['global'].log_file, + log_file=arguments['global'].log_file or '', ) except (CalledProcessError, ValueError, OSError) as error: yield from log_error_records('Error running after everything hook', error) diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index 33a6593a..c6d8c854 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -38,8 +38,13 @@ commands: - echo "Something went wrong!" ``` -If a `run:` command contains a special YAML character such as a colon, you may -need to quote the entire string (or use a [multiline +If you're coming from an older version of borgmatic, there is tooling to help +you [upgrade your +configuration](https://torsion.org/borgmatic/docs/how-to/upgrade/#upgrading-your-configuration) +to this new command hook format. + +Note that if a `run:` command contains a special YAML character such as a colon, +you may need to quote the entire string (or use a [multiline string](https://yaml-multiline.info/)) to avoid an error: ```yaml @@ -61,8 +66,8 @@ Each command in the `commands:` list has the following options: * `when`: Only trigger the hook when borgmatic is run with particular actions (`create`, `prune`, etc.) listed here. Defaults to running for all actions. * `run`: List of one or more shell commands or scripts to run when this command hook is triggered. -Note that borgmatic does not run `error` hooks if an error occurs within an -`everything` hook. +borgmatic does not run `error` hooks if an error occurs within an `everything` +hook. There's also another command hook that works a little differently: From 8a20ee73047ec96fb49e8ad18a81b0b924d8e442 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 11 Mar 2025 14:08:53 -0700 Subject: [PATCH 101/226] Fix typo in documentation (#1019). --- .../backup-to-a-removable-drive-or-an-intermittent-server.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md b/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md index b6b65a98..4517ee8b 100644 --- a/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md +++ b/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md @@ -138,7 +138,7 @@ There are some caveats you should be aware of with this feature. current repository, for instance so you can have multiple repositories in a single configuration file, have a look at [command hook variable interpolation](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/#variable-interpolation). - And there's always still the option of puting anything that you don't want + And there's always still the option of putting anything that you don't want soft-failed (like always-online cloud backups) in separate configuration files from your soft-failing repositories. * The soft failure doesn't have to test anything related to a repository. You From d3413e09070c4d35d0bd6ec3a2a8bd201ecf6064 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 11 Mar 2025 14:20:42 -0700 Subject: [PATCH 102/226] Documentation clarification (#1019). --- docs/how-to/add-preparation-and-cleanup-steps-to-backups.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index c6d8c854..ee1c1e79 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -32,7 +32,7 @@ commands: - create - prune run: - - echo "After create and/or prune!" + - echo "After create or prune!" - after: error run: - echo "Something went wrong!" From cdd0e6f05271e03258733788ffd802a8e9698637 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 11 Mar 2025 14:42:25 -0700 Subject: [PATCH 103/226] Fix incorrect kwarg in LVM hook (#790). --- borgmatic/hooks/data_source/lvm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borgmatic/hooks/data_source/lvm.py b/borgmatic/hooks/data_source/lvm.py index 7ee07296..761a5aa6 100644 --- a/borgmatic/hooks/data_source/lvm.py +++ b/borgmatic/hooks/data_source/lvm.py @@ -200,7 +200,7 @@ def dump_data_sources( ''' with borgmatic.hooks.command.Before_after_hooks( command_hooks=config.get('commands'), - function_name='dump_data_sources', + before_after='dump_data_sources', umask=config.get('umask'), dry_run=dry_run, hook_name='lvm', From f5c9bc4fa9d8beacbe3b515e86ad10638d915e1f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 11 Mar 2025 16:46:07 -0700 Subject: [PATCH 104/226] Add a "not yet released" note on 2.0.0 in docs (#790). --- docs/how-to/add-preparation-and-cleanup-steps-to-backups.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index ee1c1e79..614e1a47 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -17,9 +17,9 @@ points as it runs. feature](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/) instead.) -New in version 2.0.0 Command -hooks are now configured via a list of `commands:` in your borgmatic -configuration file. For example: +New in version 2.0.0 (not yet +released) Command hooks are now configured via a list of `commands:` in +your borgmatic configuration file. For example: ```yaml commands: From 6b6e1e0336dc298df03141c4c1f70021fa51b7ed Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 12 Mar 2025 14:13:29 -0700 Subject: [PATCH 105/226] Make the "configuration" command hook support "error" hooks and also pinging monitoring on failure (#790). --- borgmatic/commands/borgmatic.py | 424 ++++++------ borgmatic/config/paths.py | 2 +- borgmatic/hooks/command.py | 4 +- borgmatic/hooks/monitoring/cronhub.py | 2 +- borgmatic/hooks/monitoring/cronitor.py | 2 +- borgmatic/hooks/monitoring/pagerduty.py | 2 +- borgmatic/logger.py | 2 +- ...reparation-and-cleanup-steps-to-backups.md | 5 +- tests/unit/commands/test_borgmatic.py | 606 +++++++++++++----- 9 files changed, 670 insertions(+), 379 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index b7c6ad52..5691df71 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -67,6 +67,113 @@ def get_skip_actions(config, arguments): return skip_actions +class Monitoring_hooks: + ''' + A Python context manager for pinging monitoring hooks for the start state before the wrapped + code and log and finish (or failure) after the wrapped code. Also responsible for + initializing/destroying the monitoring hooks. + + Example use as a context manager: + + with Monitoring_hooks(config_filename, config, arguments, global_arguments): + do_stuff() + ''' + + def __init__(self, config_filename, config, arguments, global_arguments): + ''' + Given a configuration filename, a configuration dict, command-line arguments as an + argparse.Namespace, and global arguments as an argparse.Namespace, save relevant data points + for use below. + ''' + using_primary_action = {'create', 'prune', 'compact', 'check'}.intersection(arguments) + self.config_filename = config_filename + self.config = config + self.dry_run = global_arguments.dry_run + self.monitoring_log_level = verbosity_to_log_level(global_arguments.monitoring_verbosity) + self.monitoring_hooks_are_activated = ( + using_primary_action and self.monitoring_log_level != DISABLED + ) + + def __enter__(self): + ''' + If monitoring hooks are enabled and a primary action is in use, initialize monitoring hooks + and ping them for the "start" state. + ''' + if not self.monitoring_hooks_are_activated: + return + + dispatch.call_hooks( + 'initialize_monitor', + self.config, + dispatch.Hook_type.MONITORING, + self.config_filename, + self.monitoring_log_level, + self.dry_run, + ) + + try: + dispatch.call_hooks( + 'ping_monitor', + self.config, + dispatch.Hook_type.MONITORING, + self.config_filename, + monitor.State.START, + self.monitoring_log_level, + self.dry_run, + ) + except (OSError, CalledProcessError) as error: + raise ValueError(f'Error pinging monitor: {error}') + + def __exit__(self, exception_type, exception, traceback): + ''' + If monitoring hooks are enabled and a primary action is in use, ping monitoring hooks for + the "log" state and also the "finish" or "fail" states (depending on whether there's an + exception). Lastly, destroy monitoring hooks. + ''' + if not self.monitoring_hooks_are_activated: + return + + # Send logs irrespective of error. + try: + dispatch.call_hooks( + 'ping_monitor', + self.config, + dispatch.Hook_type.MONITORING, + self.config_filename, + monitor.State.LOG, + self.monitoring_log_level, + self.dry_run, + ) + except (OSError, CalledProcessError) as error: + raise ValueError(f'Error pinging monitor: {error}') + + try: + dispatch.call_hooks( + 'ping_monitor', + self.config, + dispatch.Hook_type.MONITORING, + self.config_filename, + monitor.State.FAIL if exception else monitor.State.FINISH, + self.monitoring_log_level, + self.dry_run, + ) + except (OSError, CalledProcessError) as error: + # If the wrapped code errored, prefer raising that exception, as it's probably more + # important than a monitor failing to ping. + if exception: + return + + raise ValueError(f'Error pinging monitor: {error}') + + dispatch.call_hooks( + 'destroy_monitor', + self.config, + dispatch.Hook_type.MONITORING, + self.monitoring_log_level, + self.dry_run, + ) + + def run_configuration(config_filename, config, config_paths, arguments): ''' Given a config filename, the corresponding parsed config dict, a sequence of loaded @@ -84,11 +191,9 @@ def run_configuration(config_filename, config, config_paths, arguments): remote_path = config.get('remote_path') retries = config.get('retries', 0) retry_wait = config.get('retry_wait', 0) + repo_queue = Queue() encountered_error = None error_repository = None - using_primary_action = {'create', 'prune', 'compact', 'check'}.intersection(arguments) - monitoring_log_level = verbosity_to_log_level(global_arguments.monitoring_verbosity) - monitoring_hooks_are_activated = using_primary_action and monitoring_log_level != DISABLED skip_actions = get_skip_actions(config, arguments) if skip_actions: @@ -96,187 +201,104 @@ def run_configuration(config_filename, config, config_paths, arguments): f"Skipping {'/'.join(skip_actions)} action{'s' if len(skip_actions) > 1 else ''} due to configured skip_actions" ) - with borgmatic.hooks.command.Before_after_hooks( - command_hooks=config.get('commands'), - before_after='configuration', - umask=config.get('umask'), - dry_run=global_arguments.dry_run, - action_names=arguments.keys(), - configuration_filename=config_filename, - log_file=arguments['global'].log_file or '', - ): - try: - local_borg_version = borg_version.local_borg_version(config, local_path) - logger.debug(f'Borg {local_borg_version}') - except (OSError, CalledProcessError, ValueError) as error: - yield from log_error_records( - f'{config_filename}: Error getting local Borg version', error - ) - return - - try: - if monitoring_hooks_are_activated: - dispatch.call_hooks( - 'initialize_monitor', - config, - dispatch.Hook_type.MONITORING, - config_filename, - monitoring_log_level, - global_arguments.dry_run, - ) - - dispatch.call_hooks( - 'ping_monitor', - config, - dispatch.Hook_type.MONITORING, - config_filename, - monitor.State.START, - monitoring_log_level, - global_arguments.dry_run, - ) - except (OSError, CalledProcessError) as error: - if command.considered_soft_failure(error): - return - - encountered_error = error - yield from log_error_records(f'{config_filename}: Error pinging monitor', error) - - if not encountered_error: - repo_queue = Queue() - for repo in config['repositories']: - repo_queue.put( - (repo, 0), - ) - - while not repo_queue.empty(): - repository, retry_num = repo_queue.get() - - with Log_prefix(repository.get('label', repository['path'])): - logger.debug('Running actions for repository') - timeout = retry_num * retry_wait - if timeout: - logger.warning(f'Sleeping {timeout}s before next retry') - time.sleep(timeout) - try: - yield from run_actions( - arguments=arguments, - config_filename=config_filename, - config=config, - config_paths=config_paths, - local_path=local_path, - remote_path=remote_path, - local_borg_version=local_borg_version, - repository=repository, - ) - except (OSError, CalledProcessError, ValueError) as error: - if retry_num < retries: - repo_queue.put( - (repository, retry_num + 1), - ) - tuple( # Consume the generator so as to trigger logging. - log_error_records( - 'Error running actions for repository', - error, - levelno=logging.WARNING, - log_command_error_output=True, - ) - ) - logger.warning(f'Retrying... attempt {retry_num + 1}/{retries}') - continue - - if command.considered_soft_failure(error): - continue - - yield from log_error_records( - 'Error running actions for repository', - error, - ) - encountered_error = error - error_repository = repository - - try: - if monitoring_hooks_are_activated: - # Send logs irrespective of error. - dispatch.call_hooks( - 'ping_monitor', - config, - dispatch.Hook_type.MONITORING, - config_filename, - monitor.State.LOG, - monitoring_log_level, - global_arguments.dry_run, - ) - except (OSError, CalledProcessError) as error: - encountered_error = error - yield from log_error_records('Error pinging monitor', error) - - if not encountered_error: - try: - if monitoring_hooks_are_activated: - dispatch.call_hooks( - 'ping_monitor', - config, - dispatch.Hook_type.MONITORING, - config_filename, - monitor.State.FINISH, - monitoring_log_level, - global_arguments.dry_run, - ) - dispatch.call_hooks( - 'destroy_monitor', - config, - dispatch.Hook_type.MONITORING, - monitoring_log_level, - global_arguments.dry_run, - ) - except (OSError, CalledProcessError) as error: - encountered_error = error - yield from log_error_records(f'{config_filename}: Error pinging monitor', error) - else: - return - - try: - command.execute_hooks( - command.filter_hooks( - config.get('commands'), after='error', action_names=arguments.keys() - ), - config.get('umask'), - global_arguments.dry_run, + try: + with Monitoring_hooks(config_filename, config, arguments, global_arguments): + with borgmatic.hooks.command.Before_after_hooks( + command_hooks=config.get('commands'), + before_after='configuration', + umask=config.get('umask'), + dry_run=global_arguments.dry_run, + action_names=arguments.keys(), configuration_filename=config_filename, log_file=arguments['global'].log_file or '', - repository=error_repository.get('path', '') if error_repository else '', - repository_label=error_repository.get('label', '') if error_repository else '', - error=encountered_error, - output=getattr(encountered_error, 'output', ''), - ) - except (OSError, CalledProcessError) as error: - if command.considered_soft_failure(error): - return + ): + try: + local_borg_version = borg_version.local_borg_version(config, local_path) + logger.debug(f'Borg {local_borg_version}') + except (OSError, CalledProcessError, ValueError) as error: + yield from log_error_records( + f'{config_filename}: Error getting local Borg version', error + ) + return - yield from log_error_records( - f'{config_filename}: Error running after error hook', error - ) + for repo in config['repositories']: + repo_queue.put( + (repo, 0), + ) - try: - if monitoring_hooks_are_activated: - dispatch.call_hooks( - 'ping_monitor', - config, - dispatch.Hook_type.MONITORING, - config_filename, - monitor.State.FAIL, - monitoring_log_level, - global_arguments.dry_run, - ) - dispatch.call_hooks( - 'destroy_monitor', - config, - dispatch.Hook_type.MONITORING, - monitoring_log_level, - global_arguments.dry_run, - ) - except (OSError, CalledProcessError) as error: - yield from log_error_records(f'{config_filename}: Error pinging monitor', error) + while not repo_queue.empty(): + repository, retry_num = repo_queue.get() + + with Log_prefix(repository.get('label', repository['path'])): + logger.debug('Running actions for repository') + timeout = retry_num * retry_wait + if timeout: + logger.warning(f'Sleeping {timeout}s before next retry') + time.sleep(timeout) + try: + yield from run_actions( + arguments=arguments, + config_filename=config_filename, + config=config, + config_paths=config_paths, + local_path=local_path, + remote_path=remote_path, + local_borg_version=local_borg_version, + repository=repository, + ) + except (OSError, CalledProcessError, ValueError) as error: + if retry_num < retries: + repo_queue.put( + (repository, retry_num + 1), + ) + tuple( # Consume the generator so as to trigger logging. + log_error_records( + 'Error running actions for repository', + error, + levelno=logging.WARNING, + log_command_error_output=True, + ) + ) + logger.warning(f'Retrying... attempt {retry_num + 1}/{retries}') + continue + + if command.considered_soft_failure(error): + continue + + yield from log_error_records( + 'Error running actions for repository', + error, + ) + encountered_error = error + error_repository = repository + + except (OSError, CalledProcessError, ValueError) as error: + yield from log_error_records('Error running configuration', error) + + encountered_error = error + + if not encountered_error: + return + + try: + command.execute_hooks( + command.filter_hooks( + config.get('commands'), after='error', action_names=arguments.keys() + ), + config.get('umask'), + global_arguments.dry_run, + configuration_filename=config_filename, + log_file=arguments['global'].log_file or '', + repository=error_repository.get('path', '') if error_repository else '', + repository_label=error_repository.get('label', '') if error_repository else '', + error=encountered_error, + output=getattr(encountered_error, 'output', ''), + ) + except (OSError, CalledProcessError) as error: + if command.considered_soft_failure(error): + return + + yield from log_error_records(f'{config_filename}: Error running after error hook', error) def run_actions( @@ -845,33 +867,25 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): for config_filename, config in configs.items(): with Log_prefix(config_filename): - try: - results = list(run_configuration(config_filename, config, config_paths, arguments)) - except (OSError, CalledProcessError, ValueError) as error: - yield from log_error_records( - 'Error running configuration file', - error, - levelno=logging.CRITICAL, - log_command_error_output=True, - ) - else: - error_logs = tuple( - result for result in results if isinstance(result, logging.LogRecord) - ) + results = list(run_configuration(config_filename, config, config_paths, arguments)) - if error_logs: - yield from log_error_records('An error occurred') - yield from error_logs - else: - yield logging.makeLogRecord( - dict( - levelno=logging.INFO, - levelname='INFO', - msg='Successfully ran configuration file', - ) + error_logs = tuple( + result for result in results if isinstance(result, logging.LogRecord) + ) + + if error_logs: + yield from log_error_records('An error occurred') + yield from error_logs + else: + yield logging.makeLogRecord( + dict( + levelno=logging.INFO, + levelname='INFO', + msg='Successfully ran configuration file', ) - if results: - json_results.extend(results) + ) + if results: + json_results.extend(results) if 'umount' in arguments: logger.info(f"Unmounting mount point {arguments['umount'].mount_point}") diff --git a/borgmatic/config/paths.py b/borgmatic/config/paths.py index 7b13d306..48dff228 100644 --- a/borgmatic/config/paths.py +++ b/borgmatic/config/paths.py @@ -134,7 +134,7 @@ class Runtime_directory: ''' return self.runtime_path - def __exit__(self, exception, value, traceback): + def __exit__(self, exception_type, exception, traceback): ''' Delete any temporary directory that was created as part of initialization. ''' diff --git a/borgmatic/hooks/command.py b/borgmatic/hooks/command.py index 5403f574..14c766ca 100644 --- a/borgmatic/hooks/command.py +++ b/borgmatic/hooks/command.py @@ -195,7 +195,7 @@ class Before_after_hooks: raise ValueError(f'Error running before {self.before_after} hook: {error}') - def __exit__(self, exception, value, traceback): + def __exit__(self, exception_type, exception, traceback): ''' Run the configured "after" command hooks that match the initialized data points. ''' @@ -215,7 +215,7 @@ class Before_after_hooks: if considered_soft_failure(error): return - raise ValueError(f'Error running before {self.before_after} hook: {error}') + raise ValueError(f'Error running after {self.before_after} hook: {error}') def considered_soft_failure(error): diff --git a/borgmatic/hooks/monitoring/cronhub.py b/borgmatic/hooks/monitoring/cronhub.py index 837d770b..8d64957d 100644 --- a/borgmatic/hooks/monitoring/cronhub.py +++ b/borgmatic/hooks/monitoring/cronhub.py @@ -28,7 +28,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev filename in any log entries. If this is a dry run, then don't actually ping anything. ''' if state not in MONITOR_STATE_TO_CRONHUB: - logger.debug(f'Ignoring unsupported monitoring {state.name.lower()} in Cronhub hook') + logger.debug(f'Ignoring unsupported monitoring state {state.name.lower()} in Cronhub hook') return dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' diff --git a/borgmatic/hooks/monitoring/cronitor.py b/borgmatic/hooks/monitoring/cronitor.py index e8169312..2a176dc1 100644 --- a/borgmatic/hooks/monitoring/cronitor.py +++ b/borgmatic/hooks/monitoring/cronitor.py @@ -28,7 +28,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev filename in any log entries. If this is a dry run, then don't actually ping anything. ''' if state not in MONITOR_STATE_TO_CRONITOR: - logger.debug(f'Ignoring unsupported monitoring {state.name.lower()} in Cronitor hook') + logger.debug(f'Ignoring unsupported monitoring state {state.name.lower()} in Cronitor hook') return dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' diff --git a/borgmatic/hooks/monitoring/pagerduty.py b/borgmatic/hooks/monitoring/pagerduty.py index d98375d9..cd63f633 100644 --- a/borgmatic/hooks/monitoring/pagerduty.py +++ b/borgmatic/hooks/monitoring/pagerduty.py @@ -46,7 +46,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev ''' if state != monitor.State.FAIL: logger.debug( - f'Ignoring unsupported monitoring {state.name.lower()} in PagerDuty hook', + f'Ignoring unsupported monitoring state {state.name.lower()} in PagerDuty hook', ) return diff --git a/borgmatic/logger.py b/borgmatic/logger.py index 07b4de76..8e327b62 100644 --- a/borgmatic/logger.py +++ b/borgmatic/logger.py @@ -256,7 +256,7 @@ class Log_prefix: self.original_prefix = get_log_prefix() set_log_prefix(self.prefix) - def __exit__(self, exception, value, traceback): + def __exit__(self, exception_type, exception, traceback): ''' Restore any original prefix. ''' diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index 614e1a47..578601e2 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -61,14 +61,11 @@ Each command in the `commands:` list has the following options: * `action` runs before each action for each repository. This replaces the deprecated `before_create`, `after_prune`, etc. * `repository` runs before or after all actions for each repository. This replaces the deprecated `before_actions` and `after_actions`. * `configuration` runs before or after all actions and repositories in the current configuration file. - * `everything` runs before or after all configuration files. This replaces the deprecated `before_everything` and `after_everything`. + * `everything` runs before or after all configuration files. Errors here do not trigger `error` hooks or the `fail` state in monitoring hooks. This replaces the deprecated `before_everything` and `after_everything`. * `error` runs after an error occurs—and it's only available for `after`. This replaces the deprecated `on_error` hook. * `when`: Only trigger the hook when borgmatic is run with particular actions (`create`, `prune`, etc.) listed here. Defaults to running for all actions. * `run`: List of one or more shell commands or scripts to run when this command hook is triggered. -borgmatic does not run `error` hooks if an error occurs within an `everything` -hook. - There's also another command hook that works a little differently: ```yaml diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index e85a6792..c31cf904 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -27,9 +27,392 @@ def test_get_skip_actions_uses_config_and_arguments(config, arguments, expected_ assert module.get_skip_actions(config, arguments) == expected_actions +def test_monitoring_hooks_with_monioring_disabled_bails(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(module.DISABLED) + flexmock(module.dispatch).should_receive('call_hooks').never() + + with module.Monitoring_hooks( + config_filename='test.yaml', + config={}, + arguments={'create': flexmock()}, + global_arguments=flexmock(monitoring_verbosity=99, dry_run=False), + ): + pass + + +def test_monitoring_hooks_with_non_primary_action_bails(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(flexmock()) + flexmock(module.dispatch).should_receive('call_hooks').never() + + with module.Monitoring_hooks( + config_filename='test.yaml', + config={}, + arguments={'extract': flexmock()}, + global_arguments=flexmock(monitoring_verbosity=99, dry_run=False), + ): + pass + + +def test_monitoring_hooks_pings_monitors(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(flexmock()) + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'initialize_monitor', + object, + object, + object, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.START, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.LOG, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.FINISH, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.FAIL, + object, + object, + ).never() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'destroy_monitor', + object, + object, + object, + object, + ).once() + + with module.Monitoring_hooks( + config_filename='test.yaml', + config={}, + arguments={'create': flexmock()}, + global_arguments=flexmock(monitoring_verbosity=99, dry_run=False), + ): + pass + + +def test_monitoring_hooks_with_start_ping_error_raises(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(flexmock()) + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'initialize_monitor', + object, + object, + object, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.START, + object, + object, + ).and_raise(OSError).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.LOG, + object, + object, + ).never() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.FINISH, + object, + object, + ).never() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'destroy_monitor', + object, + object, + object, + object, + ).never() + + with pytest.raises(ValueError): + with module.Monitoring_hooks( + config_filename='test.yaml', + config={}, + arguments={'create': flexmock()}, + global_arguments=flexmock(monitoring_verbosity=99, dry_run=False), + ): + assert False # This should never get called. + + +def test_monitoring_hooks_with_log_ping_error_raises(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(flexmock()) + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'initialize_monitor', + object, + object, + object, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.START, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.LOG, + object, + object, + ).and_raise(OSError).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.FINISH, + object, + object, + ).never() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'destroy_monitor', + object, + object, + object, + object, + ).never() + + with pytest.raises(ValueError): + with module.Monitoring_hooks( + config_filename='test.yaml', + config={}, + arguments={'create': flexmock()}, + global_arguments=flexmock(monitoring_verbosity=99, dry_run=False), + ): + pass + + +def test_monitoring_hooks_with_finish_ping_error_raises(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(flexmock()) + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'initialize_monitor', + object, + object, + object, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.START, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.LOG, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.FINISH, + object, + object, + ).and_raise(OSError).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'destroy_monitor', + object, + object, + object, + object, + ).never() + + with pytest.raises(ValueError): + with module.Monitoring_hooks( + config_filename='test.yaml', + config={}, + arguments={'create': flexmock()}, + global_arguments=flexmock(monitoring_verbosity=99, dry_run=False), + ): + pass + + +def test_monitoring_with_wrapped_code_error_pings_fail(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(flexmock()) + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'initialize_monitor', + object, + object, + object, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.START, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.LOG, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.FINISH, + object, + object, + ).never() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.FAIL, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'destroy_monitor', + object, + object, + object, + object, + ).once() + + with pytest.raises(OSError): + with module.Monitoring_hooks( + config_filename='test.yaml', + config={}, + arguments={'create': flexmock()}, + global_arguments=flexmock(monitoring_verbosity=99, dry_run=False), + ): + raise OSError() + + +def test_monitoring_with_fail_ping_error_raise_original_error(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(flexmock()) + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'initialize_monitor', + object, + object, + object, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.START, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.LOG, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.FINISH, + object, + object, + ).never() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.FAIL, + object, + object, + ).and_raise(OSError).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'destroy_monitor', + object, + object, + object, + object, + ).never() + + with pytest.raises(OSError): + with module.Monitoring_hooks( + config_filename='test.yaml', + config={}, + arguments={'create': flexmock()}, + global_arguments=flexmock(monitoring_verbosity=99, dry_run=False), + ): + raise OSError() + + def test_run_configuration_runs_actions_for_each_repository(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) expected_results = [flexmock(), flexmock()] @@ -48,6 +431,7 @@ def test_run_configuration_runs_actions_for_each_repository(): def test_run_configuration_with_skip_actions_does_not_raise(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return(['compact']) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module).should_receive('Log_prefix').and_return(flexmock()) @@ -61,9 +445,9 @@ def test_run_configuration_with_skip_actions_does_not_raise(): def test_run_configuration_with_invalid_borg_version_errors(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_raise(ValueError) - flexmock(module.dispatch).should_receive('call_hooks').never() flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').never() config = {'repositories': [{'path': 'foo'}]} @@ -75,58 +459,12 @@ def test_run_configuration_with_invalid_borg_version_errors(): list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) -def test_run_configuration_logs_monitor_start_error(): - flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) - flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) - flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.dispatch).should_receive('call_hooks').and_raise(OSError).and_return( - None - ).and_return(None).and_return(None) - expected_results = [flexmock()] - flexmock(module).should_receive('log_error_records').and_return(expected_results) - flexmock(module).should_receive('Log_prefix').and_return(flexmock()) - flexmock(module).should_receive('run_actions').never() - flexmock(module.command).should_receive('filter_hooks') - flexmock(module.command).should_receive('execute_hooks') - config = {'repositories': [{'path': 'foo'}]} - arguments = { - 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), - 'create': flexmock(), - } - - results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) - - assert results == expected_results - - -def test_run_configuration_bails_for_monitor_start_soft_failure(): - flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) - flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) - flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') - flexmock(module.dispatch).should_receive('call_hooks').and_raise(error).and_return(None) - flexmock(module).should_receive('log_error_records').never() - flexmock(module).should_receive('Log_prefix').and_return(flexmock()) - flexmock(module).should_receive('run_actions').never() - config = {'repositories': [{'path': 'foo'}, {'path': 'bar'}]} - arguments = { - 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), - 'create': flexmock(), - } - - results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) - - assert results == [] - - def test_run_configuration_logs_actions_error(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.dispatch).should_receive('call_hooks') expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('Log_prefix').and_return(flexmock()) @@ -144,9 +482,9 @@ def test_run_configuration_logs_actions_error(): def test_run_configuration_skips_remaining_actions_for_actions_soft_failure_but_still_runs_next_repository_actions(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.dispatch).should_receive('call_hooks').times(5) error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') log = flexmock() flexmock(module).should_receive('Log_prefix').and_return(flexmock()) @@ -164,106 +502,13 @@ def test_run_configuration_skips_remaining_actions_for_actions_soft_failure_but_ assert results == [log] -def test_run_configuration_logs_monitor_log_error(): - flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) - flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) - flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return( - None - ).and_raise(OSError) - expected_results = [flexmock()] - flexmock(module).should_receive('log_error_records').and_return(expected_results) - flexmock(module).should_receive('Log_prefix').and_return(flexmock()) - flexmock(module).should_receive('run_actions').and_return([]) - flexmock(module.command).should_receive('filter_hooks') - flexmock(module.command).should_receive('execute_hooks') - config = {'repositories': [{'path': 'foo'}]} - arguments = { - 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), - 'create': flexmock(), - } - - results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) - - assert results == expected_results - - -def test_run_configuration_logs_monitor_finish_error(): - flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) - flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) - flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return( - None - ).and_return(None).and_raise(OSError) - expected_results = [flexmock()] - flexmock(module).should_receive('log_error_records').and_return(expected_results) - flexmock(module).should_receive('Log_prefix').and_return(flexmock()) - flexmock(module).should_receive('run_actions').and_return([]) - flexmock(module.command).should_receive('filter_hooks') - flexmock(module.command).should_receive('execute_hooks') - config = {'repositories': [{'path': 'foo'}]} - arguments = { - 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), - 'create': flexmock(), - } - - results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) - - assert results == expected_results - - -def test_run_configuration_logs_monitor_fail_error(): - flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) - flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) - flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.dispatch).should_receive('call_hooks') - - # Trigger an error in the monitor finish so that the monitor fail also gets triggered. - flexmock(module.dispatch).should_receive('call_hooks').with_args( - 'ping_monitor', - object, - module.dispatch.Hook_type.MONITORING, - object, - module.monitor.State.FINISH, - object, - object, - ).and_raise(OSError) - flexmock(module.dispatch).should_receive('call_hooks').with_args( - 'ping_monitor', - object, - module.dispatch.Hook_type.MONITORING, - object, - module.monitor.State.FAIL, - object, - object, - ).and_raise(OSError).once() - expected_results = [flexmock()] - flexmock(module).should_receive('log_error_records').and_return(expected_results) - flexmock(module).should_receive('Log_prefix').and_return(flexmock()) - flexmock(module).should_receive('run_actions').and_return([]) - flexmock(module.command).should_receive('filter_hooks') - flexmock(module.command).should_receive('execute_hooks') - config = {'repositories': [{'path': 'foo'}]} - arguments = { - 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), - 'create': flexmock(), - } - - results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) - - assert results == expected_results + expected_results - - def test_run_configuration_does_not_call_monitoring_hooks_if_monitoring_hooks_are_disabled(): flexmock(module).should_receive('verbosity_to_log_level').and_return(module.DISABLED) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.dispatch).should_receive('call_hooks').never() flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_return([]) @@ -279,6 +524,7 @@ def test_run_configuration_does_not_call_monitoring_hooks_if_monitoring_hooks_ar def test_run_configuration_logs_on_error_hook_error(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.command).should_receive('filter_hooks') @@ -300,9 +546,56 @@ def test_run_configuration_logs_on_error_hook_error(): assert results == expected_results +def test_run_configuration_logs_on_before_command_hook_error(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) + flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) + flexmock(module.command).should_receive('Before_after_hooks').and_raise(OSError) + flexmock(module.borg_version).should_receive('local_borg_version').never() + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') + expected_results = [flexmock()] + flexmock(module).should_receive('log_error_records').and_return(expected_results) + flexmock(module).should_receive('Log_prefix').never() + flexmock(module).should_receive('run_actions').never() + config = {'repositories': [{'path': 'foo'}]} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } + + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) + + assert results == expected_results + + +def test_run_configuration_logs_on_monitoring_hook_error(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) + flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_raise(OSError) + flexmock(module.command).should_receive('Before_after_hooks').never() + flexmock(module.borg_version).should_receive('local_borg_version').never() + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') + expected_results = [flexmock()] + flexmock(module).should_receive('log_error_records').and_return(expected_results) + flexmock(module).should_receive('Log_prefix').never() + flexmock(module).should_receive('run_actions').never() + config = {'repositories': [{'path': 'foo'}]} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } + + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) + + assert results == expected_results + + def test_run_configuration_bails_for_on_error_hook_soft_failure(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') @@ -327,6 +620,7 @@ def test_run_configuration_retries_soft_error(): # Run action first fails, second passes. flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module).should_receive('Log_prefix').and_return(flexmock()) @@ -349,6 +643,7 @@ def test_run_configuration_retries_hard_error(): # Run action fails twice. flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module).should_receive('Log_prefix').and_return(flexmock()) @@ -380,6 +675,7 @@ def test_run_configuration_retries_hard_error(): def test_run_configuration_retries_repositories_in_order(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module).should_receive('Log_prefix').and_return(flexmock()) @@ -407,6 +703,7 @@ def test_run_configuration_retries_repositories_in_order(): def test_run_configuration_retries_round_robin(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module).should_receive('Log_prefix').and_return(flexmock()) @@ -450,6 +747,7 @@ def test_run_configuration_retries_round_robin(): def test_run_configuration_with_one_retry(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module).should_receive('Log_prefix').and_return(flexmock()) @@ -491,6 +789,7 @@ def test_run_configuration_with_one_retry(): def test_run_configuration_with_retry_wait_does_backoff_after_each_retry(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module).should_receive('Log_prefix').and_return(flexmock()) @@ -543,6 +842,7 @@ def test_run_configuration_with_retry_wait_does_backoff_after_each_retry(): def test_run_configuration_with_multiple_repositories_retries_with_timeout(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module).should_receive('Log_prefix').and_return(flexmock()) @@ -1715,26 +2015,6 @@ def test_collect_configuration_run_summary_logs_run_configuration_error_logs(): assert {log.levelno for log in logs} == {logging.CRITICAL} -def test_collect_configuration_run_summary_logs_run_configuration_exception(): - flexmock(module.validate).should_receive('guard_configuration_contains_repository') - flexmock(module.command).should_receive('filter_hooks') - flexmock(module.command).should_receive('execute_hooks') - flexmock(module).should_receive('Log_prefix').and_return(flexmock()) - flexmock(module).should_receive('run_configuration').and_raise(ValueError) - flexmock(module).should_receive('log_error_records').and_return( - [flexmock(levelno=logging.CRITICAL)] - ) - arguments = {'global': flexmock(dry_run=False, log_file=flexmock())} - - logs = tuple( - module.collect_configuration_run_summary_logs( - {'test.yaml': {}}, config_paths=['/tmp/test.yaml'], arguments=arguments - ) - ) - - assert {log.levelno for log in logs} == {logging.CRITICAL} - - def test_collect_configuration_run_summary_logs_run_umount_error(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') flexmock(module.command).should_receive('filter_hooks') From bcb224a24361b7edfd5085b1714cc200f38433e3 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 12 Mar 2025 14:31:13 -0700 Subject: [PATCH 106/226] Claim another implemented ticket in NEWS (#821). --- NEWS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index a3a2852f..f884a3a8 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,6 @@ 2.0.0.dev0 - * #790: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more flexible - "commands:". See the documentation for more information: + * #790, #821: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more + flexible "commands:". See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ 1.9.14 From 901e668c760f6a17e4f02ef418ff65285bbca5ed Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 12 Mar 2025 17:10:35 -0700 Subject: [PATCH 107/226] Document a database use case involving a temporary database client container (#1020). --- NEWS | 2 ++ docs/how-to/backup-your-databases.md | 46 ++++++++++++++++++++-------- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/NEWS b/NEWS index f884a3a8..45b83c97 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,8 @@ * #790, #821: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more flexible "commands:". See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ + * #1020: Document a database use case involving a temporary database client container: + https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers 1.9.14 * #409: With the PagerDuty monitoring hook, send borgmatic logs to PagerDuty so they show up in the diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index 30a3590f..d4acb967 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -193,14 +193,14 @@ mysql_databases: ### Containers -If your database is running within a container and borgmatic is too, no +If your database server is running within a container and borgmatic is too, no problem—configure borgmatic to connect to the container's name on its exposed port. For instance: ```yaml postgresql_databases: - name: users - hostname: your-database-container-name + hostname: your-database-server-container-name port: 5433 username: postgres password: trustsome1 @@ -210,21 +210,22 @@ postgresql_databases: these options in the `hooks:` section of your configuration. But what if borgmatic is running on the host? You can still connect to a -database container if its ports are properly exposed to the host. For +database server container if its ports are properly exposed to the host. For instance, when running the database container, you can specify `--publish 127.0.0.1:5433:5432` so that it exposes the container's port 5432 to port 5433 -on the host (only reachable on localhost, in this case). Or the same thing -with Docker Compose: +on the host (only reachable on localhost, in this case). Or the same thing with +Docker Compose: ```yaml services: - your-database-container-name: + your-database-server-container-name: image: postgres ports: - 127.0.0.1:5433:5432 ``` -And then you can connect to the database from borgmatic running on the host: +And then you can configure borgmatic running on the host to connect to the +database: ```yaml hooks: @@ -240,9 +241,9 @@ Alter the ports in these examples to suit your particular database system. Normally, borgmatic dumps a database by running a database dump command (e.g. `pg_dump`) on the host or wherever borgmatic is running, and this command -connects to your containerized database via the given `hostname` and `port`. -But if you don't have any database dump commands installed on your host and -you'd rather use the commands inside your database container itself, borgmatic +connects to your containerized database via the given `hostname` and `port`. But +if you don't have any database dump commands installed on your host and you'd +rather use the commands inside your running database container itself, borgmatic supports that too. For that, configure borgmatic to `exec` into your container to run the dump command. @@ -259,9 +260,10 @@ hooks: pg_dump_command: docker exec my_pg_container pg_dump ``` -... where `my_pg_container` is the name of your database container. In this -example, you'd also need to set the `pg_restore_command` and `psql_command` -options. +... where `my_pg_container` is the name of your running database container. +Running `pg_dump` this way takes advantage of the localhost "trust" +authentication within that container. In this example, you'd also need to set +the `pg_restore_command` and `psql_command` options. If you choose to use the `pg_dump` command within the container, and you're using the `directory` format in particular, you'll also need to mount the @@ -280,6 +282,24 @@ services: - /run/user/1000:/run/user/1000 ``` +Another variation: If you're running borgmatic on the host but want to spin up a +temporary `pg_dump` container whenever borgmatic dumps a database, for +instance to make use of a `pg_dump` version not present on the host, try +something like this: + +```yaml +hooks: + postgresql_databases: + - name: users + hostname: your-database-hostname + username: postgres + password: trustsome1 + pg_dump_command: docker run --rm --env PGPASSWORD postgres:17-alpine pg_dump +``` + +The `--env PGPASSWORD` is necessary here for borgmatic to provide your database +password to the temporary `pg_dump` container. + Similar command override options are available for (some of) the other supported database types as well. See the [configuration reference](https://torsion.org/borgmatic/docs/reference/configuration/) for From 1b4c94ad1e8ee2e1520f63e75ae9219e6b88997e Mon Sep 17 00:00:00 2001 From: Nish_ <120EE0980@nitrkl.ac.in> Date: Fri, 14 Mar 2025 10:59:01 +0530 Subject: [PATCH 108/226] Add feature toggle to pass --stats to prune on Borg 1, but not Borg 2 Signed-off-by: Nish_ <120EE0980@nitrkl.ac.in> --- borgmatic/borg/feature.py | 2 + borgmatic/borg/prune.py | 8 +++- borgmatic/commands/arguments.py | 2 +- tests/unit/borg/test_prune.py | 66 +++++++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 2 deletions(-) diff --git a/borgmatic/borg/feature.py b/borgmatic/borg/feature.py index a6462e13..c45899c0 100644 --- a/borgmatic/borg/feature.py +++ b/borgmatic/borg/feature.py @@ -17,6 +17,7 @@ class Feature(Enum): MATCH_ARCHIVES = 11 EXCLUDED_FILES_MINUS = 12 ARCHIVE_SERIES = 13 + NO_PRUNE_STATS = 14 FEATURE_TO_MINIMUM_BORG_VERSION = { @@ -33,6 +34,7 @@ FEATURE_TO_MINIMUM_BORG_VERSION = { Feature.MATCH_ARCHIVES: parse('2.0.0b3'), # borg --match-archives Feature.EXCLUDED_FILES_MINUS: parse('2.0.0b5'), # --list --filter uses "-" for excludes Feature.ARCHIVE_SERIES: parse('2.0.0b11'), # identically named archives form a series + Feature.NO_PRUNE_STATS: parse('2.0.0b10'), # prune --stats is not available } diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index fa43d5e0..82a78201 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -75,7 +75,13 @@ def prune_archives( + (('--umask', str(umask)) if umask else ()) + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) - + (('--stats',) if prune_arguments.stats and not dry_run else ()) + + ( + ('--stats',) + if prune_arguments.stats + and not dry_run + and not feature.available(feature.Feature.NO_PRUNE_STATS, local_borg_version) + else () + ) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + flags.make_flags_from_arguments( prune_arguments, diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index e95ec5f4..b024de11 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -547,7 +547,7 @@ def make_parsers(): dest='stats', default=False, action='store_true', - help='Display statistics of the pruned archive', + help='Display statistics of the pruned archive [Borg 1 only]', ) prune_group.add_argument( '--list', dest='list_archives', action='store_true', help='List archives kept/pruned' diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index 556d695f..3f2da33a 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -210,6 +210,9 @@ def test_prune_archives_calls_borg_with_flags(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('repo',), logging.INFO) prune_arguments = flexmock(stats=False, list_archives=False) @@ -228,6 +231,9 @@ def test_prune_archives_with_log_info_calls_borg_with_info_flag(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--info', 'repo'), logging.INFO) insert_logging_mock(logging.INFO) @@ -247,6 +253,9 @@ def test_prune_archives_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--debug', '--show-rc', 'repo'), logging.INFO) insert_logging_mock(logging.DEBUG) @@ -266,6 +275,9 @@ def test_prune_archives_with_dry_run_calls_borg_with_dry_run_flag(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--dry-run', 'repo'), logging.INFO) prune_arguments = flexmock(stats=False, list_archives=False) @@ -284,6 +296,9 @@ def test_prune_archives_with_local_path_calls_borg_via_local_path(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock(('borg1',) + PRUNE_COMMAND[1:] + ('repo',), logging.INFO) prune_arguments = flexmock(stats=False, list_archives=False) @@ -303,6 +318,9 @@ def test_prune_archives_with_exit_codes_calls_borg_using_them(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) borg_exit_codes = flexmock() insert_execute_command_mock( ('borg',) + PRUNE_COMMAND[1:] + ('repo',), @@ -326,6 +344,9 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_flags(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--remote-path', 'borg1', 'repo'), logging.INFO) prune_arguments = flexmock(stats=False, list_archives=False) @@ -345,6 +366,9 @@ def test_prune_archives_with_stats_calls_borg_with_stats_flag_and_answer_output_ flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--stats', 'repo'), module.borgmatic.logger.ANSWER) prune_arguments = flexmock(stats=True, list_archives=False) @@ -363,6 +387,9 @@ def test_prune_archives_with_files_calls_borg_with_list_flag_and_answer_output_l flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--list', 'repo'), module.borgmatic.logger.ANSWER) prune_arguments = flexmock(stats=False, list_archives=True) @@ -382,6 +409,9 @@ def test_prune_archives_with_umask_calls_borg_with_umask_flags(): config = {'umask': '077'} flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--umask', '077', 'repo'), logging.INFO) prune_arguments = flexmock(stats=False, list_archives=False) @@ -400,6 +430,9 @@ def test_prune_archives_with_log_json_calls_borg_with_log_json_flag(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--log-json', 'repo'), logging.INFO) prune_arguments = flexmock(stats=False, list_archives=False) @@ -419,6 +452,9 @@ def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): config = {'lock_wait': 5} flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--lock-wait', '5', 'repo'), logging.INFO) prune_arguments = flexmock(stats=False, list_archives=False) @@ -437,6 +473,9 @@ def test_prune_archives_with_extra_borg_options_calls_borg_with_extra_options(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--extra', '--options', 'repo'), logging.INFO) prune_arguments = flexmock(stats=False, list_archives=False) @@ -471,6 +510,9 @@ def test_prune_archives_with_date_based_matching_calls_borg_with_date_based_flag ) ) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) flexmock(module.environment).should_receive('make_environment') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command').with_args( @@ -521,6 +563,9 @@ def test_prune_archives_calls_borg_with_working_directory(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock( PRUNE_COMMAND + ('repo',), logging.INFO, working_directory='/working/dir' ) @@ -534,3 +579,24 @@ def test_prune_archives_calls_borg_with_working_directory(): global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, ) + + +def test_prune_archives_calls_borg_with_flags_and_when_feature_available(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '2.0.0b10' + ).and_return(True) + insert_execute_command_mock(PRUNE_COMMAND + ('repo',), logging.ANSWER) + + prune_arguments = flexmock(stats=True, list_archives=False) + module.prune_archives( + dry_run=False, + repository_path='repo', + config={}, + local_borg_version='2.0.0b10', + global_arguments=flexmock(log_json=False), + prune_arguments=prune_arguments, + ) From 92279d3c719a3b075e35e112e32e31435e6c77ca Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 14 Mar 2025 22:59:43 -0700 Subject: [PATCH 109/226] Initial work on command-line flags for all configuration (#303). --- borgmatic/commands/arguments.py | 167 ++++++++++++++++++++++++++++++-- borgmatic/commands/borgmatic.py | 30 ++++-- borgmatic/config/arguments.py | 137 ++++++++++++++++++++++++++ borgmatic/config/override.py | 7 ++ borgmatic/config/schema.yaml | 5 + borgmatic/config/validate.py | 15 +-- 6 files changed, 337 insertions(+), 24 deletions(-) create mode 100644 borgmatic/config/arguments.py diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index e95ec5f4..b09c5660 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1,5 +1,7 @@ import collections import itertools +import json +import re import sys from argparse import ArgumentParser @@ -282,12 +284,155 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse ) -def make_parsers(): +def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names=None): ''' - Build a global arguments parser, individual action parsers, and a combined parser containing - both. Return them as a tuple. The global parser is useful for parsing just global arguments - while ignoring actions, and the combined parser is handy for displaying help that includes - everything: global flags, a list of actions, etc. + Given an argparse._ArgumentGroup instance, a configuration schema dict, and a sequence of + unparsed argument strings, convert the entire schema into corresponding command-line flags and + add them to the arguments group. + + For instance, given a schema of: + + { + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'object', + 'properties': { + 'bar': {'type': 'integer'} + } + } + } + } + + ... the following flag will be added to the arguments group: + + --foo.bar + + If "foo" is instead an array of objects, it will get added like this + + --foo[0].bar + + And if names are also passed in, they are considered to be the name components of an option + (e.g. "foo" and "bar") and are used to construct a resulting flag. + ''' + if names is None: + names = () + + schema_type = schema.get('type') + + # If this option has multiple types, just use the first one (that isn't "null"). + if isinstance(schema_type, list): + try: + schema_type = next(single_type for single_type in schema_type if single_type != 'null') + except StopIteration: + raise ValueError(f'Unknown type in configuration schema: {schema_type}') + + # If this is an "object" type, recurse for each child option ("property"). + if schema_type in {'object', 'array'}: + properties = ( + schema.get('items', {}).get('properties') + if schema_type == 'array' + else schema.get('properties') + ) + + if properties: + for name, child in properties.items(): + add_arguments_from_schema( + arguments_group, + child, + unparsed_arguments, + names + ((name + '[0]',) if child.get('type') == 'array' else (name,)), + ) + + return + + flag_name = '.'.join(names) + description = schema.get('description') + metavar = names[-1].upper() + + if schema_type == 'array': + metavar = metavar.rstrip('S') + + if description: + if schema_type == 'array': + items_schema = schema.get('items', {}) + + description += ' Can specify flag multiple times.' + + if '[0]' in flag_name: + description += ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).' + + description = description.replace('%', '%%') + + try: + argument_type = {'string': str, 'integer': int, 'boolean': bool, 'array': str}[schema_type] + except KeyError: + raise ValueError(f'Unknown type in configuration schema: {schema_type}') + + arguments_group.add_argument( + f"--{flag_name.replace('_', '-')}", + type=argument_type, + metavar=metavar, + action='append' if schema_type == 'array' else None, + help=description, + ) + + # We want to support flags that can have arbitrary indices like: + # + # --foo.bar[1].baz + # + # But argparse doesn't support that natively because the index can be an arbitrary number. We + # won't let that stop us though, will we? So, if the current flag name has an array component in + # it (e.g. a name with "[0]"), then make a pattern that would match the flag name regardless of + # the number that's in it. The idea is that we want to look for unparsed arguments that appear + # like the flag name, but instead of "[0]" they have, say, "[1]" or "[123]". + # + # Next, we check each unparsed argument against that pattern. If one of them matches, add an + # argument flag for it to the argument parser group. Example: + # + # Let's say flag_name is: + # + # --foo.bar[0].baz + # + # ... then the regular expression pattern will be: + # + # ^--foo\.bar\[\d+\]\.baz + # + # ... and, if that matches an unparsed argument of: + # + # --foo.bar[1].baz + # + # ... then an argument flag will get added equal to that unparsed argument. And the unparsed + # argument will match it when parsing is performed! In this manner, we're using the actual user + # CLI input to inform what exact flags we support! + if '[0]' not in flag_name or '--help' in unparsed_arguments: + return + + pattern = re.compile(f'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.")}$') + existing_flags = set( + itertools.chain( + *(group_action.option_strings for group_action in arguments_group._group_actions) + ) + ) + + for unparsed in unparsed_arguments: + unparsed_flag_name = unparsed.split('=', 1)[0] + + if pattern.match(unparsed_flag_name) and unparsed_flag_name not in existing_flags: + arguments_group.add_argument( + unparsed_flag_name, + type=argument_type, + metavar=metavar, + help=description, + ) + + +def make_parsers(schema, unparsed_arguments): + ''' + Given a configuration schema dict, build a global arguments parser, individual action parsers, + and a combined parser containing both. Return them as a tuple. The global parser is useful for + parsing just global arguments while ignoring actions, and the combined parser is handy for + displaying help that includes everything: global flags, a list of actions, etc. ''' config_paths = collect.get_default_config_paths(expand_home=True) unexpanded_config_paths = collect.get_default_config_paths(expand_home=False) @@ -388,6 +533,7 @@ def make_parsers(): action='store_true', help='Display installed version number of borgmatic and exit', ) + add_arguments_from_schema(global_group, schema, unparsed_arguments) global_plus_action_parser = ArgumentParser( description=''' @@ -1523,15 +1669,18 @@ def make_parsers(): return global_parser, action_parsers, global_plus_action_parser -def parse_arguments(*unparsed_arguments): +def parse_arguments(schema, *unparsed_arguments): ''' - Given command-line arguments with which this script was invoked, parse the arguments and return - them as a dict mapping from action name (or "global") to an argparse.Namespace instance. + Given a configuration schema dict and the command-line arguments with which this script was + invoked, parse the arguments and return them as a dict mapping from action name (or "global") to + an argparse.Namespace instance. Raise ValueError if the arguments cannot be parsed. Raise SystemExit with an error code of 0 if "--help" was requested. ''' - global_parser, action_parsers, global_plus_action_parser = make_parsers() + global_parser, action_parsers, global_plus_action_parser = make_parsers( + schema, unparsed_arguments + ) arguments, remaining_action_arguments = parse_arguments_for_actions( unparsed_arguments, action_parsers.choices, global_parser ) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 5691df71..9cffbdf8 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -8,6 +8,8 @@ import time from queue import Queue from subprocess import CalledProcessError +import ruamel.yaml + import borgmatic.actions.borg import borgmatic.actions.break_lock import borgmatic.actions.change_passphrase @@ -33,6 +35,7 @@ import borgmatic.actions.restore import borgmatic.actions.transfer import borgmatic.commands.completion.bash import borgmatic.commands.completion.fish +import borgmatic.config.load from borgmatic.borg import umount as borg_umount from borgmatic.borg import version as borg_version from borgmatic.commands.arguments import parse_arguments @@ -570,14 +573,14 @@ def run_actions( ) -def load_configurations(config_filenames, overrides=None, resolve_env=True): +def load_configurations(config_filenames, global_arguments, overrides=None, resolve_env=True): ''' - Given a sequence of configuration filenames, a sequence of configuration file override strings - in the form of "option.suboption=value", and whether to resolve environment variables, load and - validate each configuration file. Return the results as a tuple of: dict of configuration - filename to corresponding parsed configuration, a sequence of paths for all loaded configuration - files (including includes), and a sequence of logging.LogRecord instances containing any parse - errors. + Given a sequence of configuration filenames, global arguments as an argparse.Namespace, a + sequence of configuration file override strings in the form of "option.suboption=value", and + whether to resolve environment variables, load and validate each configuration file. Return the + results as a tuple of: dict of configuration filename to corresponding parsed configuration, a + sequence of paths for all loaded configuration files (including includes), and a sequence of + logging.LogRecord instances containing any parse errors. Log records are returned here instead of being logged directly because logging isn't yet initialized at this point! (Although with the Delayed_logging_handler now in place, maybe this @@ -605,6 +608,7 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True): configs[config_filename], paths, parse_logs = validate.parse_configuration( config_filename, validate.schema_filename(), + global_arguments, overrides, resolve_env, ) @@ -928,9 +932,17 @@ def exit_with_help_link(): # pragma: no cover def main(extra_summary_logs=[]): # pragma: no cover configure_signals() configure_delayed_logging() + schema_filename = validate.schema_filename() try: - arguments = parse_arguments(*sys.argv[1:]) + schema = borgmatic.config.load.load_configuration(schema_filename) + except (ruamel.yaml.error.YAMLError, RecursionError) as error: + configure_logging(logging.CRITICAL) + logger.critical(error) + exit_with_help_link() + + try: + arguments = parse_arguments(schema, *sys.argv[1:]) except ValueError as error: configure_logging(logging.CRITICAL) logger.critical(error) @@ -953,10 +965,10 @@ def main(extra_summary_logs=[]): # pragma: no cover print(borgmatic.commands.completion.fish.fish_completion()) sys.exit(0) - validate = bool('validate' in arguments) config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths)) configs, config_paths, parse_logs = load_configurations( config_filenames, + global_arguments, global_arguments.overrides, resolve_env=global_arguments.resolve_env and not validate, ) diff --git a/borgmatic/config/arguments.py b/borgmatic/config/arguments.py new file mode 100644 index 00000000..96a28b4d --- /dev/null +++ b/borgmatic/config/arguments.py @@ -0,0 +1,137 @@ +import io +import re + +import ruamel.yaml + + +LIST_INDEX_KEY_PATTERN = re.compile(r'^(?P[a-zA-z-]+)\[(?P\d+)\]$') + + +def set_values(config, keys, value): + ''' + Given a configuration dict, a sequence of parsed key strings, and a string value, descend into + the configuration hierarchy based on the keys to set the value into the right place. + ''' + if not keys: + return + + first_key = keys[0] + + # Support "name[0]"-style list index syntax. + match = LIST_INDEX_KEY_PATTERN.match(first_key) + + if match: + list_key = match.group('list_name') + list_index = int(match.group('index')) + + if len(keys) == 1: + config[list_key][list_index] = value + + return + + if list_key not in config: + config[list_key] = [] + + try: + set_values(config[list_key][list_index], keys[1:], value) + except IndexError: + raise ValueError(f'The list index {first_key} is out of range') + + return + + if len(keys) == 1: + config[first_key] = value + return + + if first_key not in config: + config[first_key] = {} + + set_values(config[first_key], keys[1:], value) + + +def type_for_option(schema, option_keys): + ''' + Given a configuration schema dict and a sequence of keys identifying a potentially nested + option, e.g. ('extra_borg_options', 'create'), return the schema type of that option as a + string. + + Return None if the option or its type cannot be found in the schema. + ''' + option_schema = schema + + for key in option_keys: + # Support "name[0]"-style list index syntax. + match = LIST_INDEX_KEY_PATTERN.match(key) + + try: + if match: + option_schema = option_schema['properties'][match.group('list_name')]['items'] + else: + option_schema = option_schema['properties'][key] + except KeyError: + return None + + try: + return option_schema['type'] + except KeyError: + return None + + +def prepare_arguments_for_config(global_arguments, schema): + ''' + Given global arguments as an argparse.Namespace and a configuration schema dict, parse each + argument that corresponds to an option in the schema and return a sequence of tuples (keys, + values) for that option, where keys is a sequence of strings. For instance, given the following + arguments: + + argparse.Namespace(**{'my_option.sub_option': 'value1', 'other_option': 'value2'}) + + ... return this: + + ( + (('my_option', 'sub_option'), 'value1'), + (('other_option'), 'value2'), + ) + + Raise ValueError if an override can't be parsed. + ''' + prepared_values = [] + + for argument_name, value in global_arguments.__dict__.items(): + try: + if value is None: + continue + + keys = tuple(argument_name.split('.')) + option_type = type_for_option(schema, keys) + + # The argument doesn't correspond to any option in the schema, so ignore it. It's + # probably a flag that borgmatic has on the command-line but not in configuration. + if option_type is None: + continue + + prepared_values.append( + ( + keys, + value, + ) + ) + except ruamel.yaml.error.YAMLError as error: + raise ValueError(f"Invalid override '{raw_override}': {error.problem}") + + return tuple(prepared_values) + + +def apply_arguments_to_config(config, schema, global_arguments): + ''' + Given a configuration dict, a corresponding configuration schema dict, and global arguments as + an argparse.Namespace, set those given argument values into their corresponding configuration + options in the configuration dict. + + This supports argument flags of the from "--foo.bar.baz" where each dotted component is a nested + configuration object. Additionally, flags like "--foo.bar[0].baz" are supported to update a list + element in the configuration. + ''' + + for keys, value in prepare_arguments_for_config(global_arguments, schema): + set_values(config, keys, value) diff --git a/borgmatic/config/override.py b/borgmatic/config/override.py index 8a60cb50..0893f669 100644 --- a/borgmatic/config/override.py +++ b/borgmatic/config/override.py @@ -1,8 +1,12 @@ import io +import logging import ruamel.yaml +logger = logging.getLogger(__name__) + + def set_values(config, keys, value): ''' Given a hierarchy of configuration dicts, a sequence of parsed key strings, and a string value, @@ -134,6 +138,9 @@ def apply_overrides(config, schema, raw_overrides): ''' overrides = parse_overrides(raw_overrides, schema) + if overrides: + logger.warning("The --override flag is deprecated and will be removed from a future release. Instead, use a command-line flag corresponding to the configuration option you'd like to set.") + for keys, value in overrides: set_values(config, keys, value) set_values(config, strip_section_names(keys), value) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 13ab0686..3c1bb20f 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -36,9 +36,14 @@ properties: properties: path: type: string + description: The local path or Borg URL of the repository. example: ssh://user@backupserver/./{fqdn} label: type: string + description: | + An optional label for the repository, used in logging + and to make selecting the repository easier on the + command-line. example: backupserver description: | A required list of local or remote repositories with paths and diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index c80cee4d..35c8e809 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -4,7 +4,7 @@ import os import jsonschema import ruamel.yaml -import borgmatic.config +import borgmatic.config.arguments from borgmatic.config import constants, environment, load, normalize, override @@ -84,13 +84,15 @@ def apply_logical_validation(config_filename, parsed_configuration): ) -def parse_configuration(config_filename, schema_filename, overrides=None, resolve_env=True): +def parse_configuration(config_filename, schema_filename, global_arguments, overrides=None, resolve_env=True): ''' Given the path to a config filename in YAML format, the path to a schema filename in a YAML - rendition of JSON Schema format, a sequence of configuration file override strings in the form - of "option.suboption=value", and whether to resolve environment variables, return the parsed - configuration as a data structure of nested dicts and lists corresponding to the schema. Example - return value: + rendition of JSON Schema format, global arguments as an argparse.Namespace, a sequence of + configuration file override strings in the form of "option.suboption=value", and whether to + resolve environment variables, return the parsed configuration as a data structure of nested + dicts and lists corresponding to the schema. Example return value. + + Example return value: { 'source_directories': ['/home', '/etc'], @@ -113,6 +115,7 @@ def parse_configuration(config_filename, schema_filename, overrides=None, resolv except (ruamel.yaml.error.YAMLError, RecursionError) as error: raise Validation_error(config_filename, (str(error),)) + borgmatic.config.arguments.apply_arguments_to_config(config, schema, global_arguments) override.apply_overrides(config, schema, overrides) constants.apply_constants(config, config.get('constants') if config else {}) From 1c92d84e0902c905098866dc8df9803bda905f59 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 15 Mar 2025 10:02:47 -0700 Subject: [PATCH 110/226] Add Borg 2 "prune --stats" flag change to NEWS (#1010). --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index 45b83c97..b3d053fb 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,7 @@ * #790, #821: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more flexible "commands:". See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ + * #1010: When using Borg 2, don't pass the "--stats" flag to "borg prune". * #1020: Document a database use case involving a temporary database client container: https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers From c84815bfb040a8ed110a8eb293ffa93be7ec6af1 Mon Sep 17 00:00:00 2001 From: Nish_ <120EE0980@nitrkl.ac.in> Date: Sat, 15 Mar 2025 22:16:57 +0530 Subject: [PATCH 111/226] add custom dump and restore commands for sqlite hook Signed-off-by: Nish_ <120EE0980@nitrkl.ac.in> --- borgmatic/config/schema.yaml | 18 +++ borgmatic/hooks/data_source/sqlite.py | 16 ++- tests/unit/hooks/data_source/test_sqlite.py | 146 ++++++++++++++++++++ 3 files changed, 174 insertions(+), 6 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 13ab0686..dbd79d0c 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1642,6 +1642,24 @@ properties: Path to the SQLite database file to restore to. Defaults to the "path" option. example: /var/lib/sqlite/users.db + sqlite_command: + type: string + description: | + Command to use instead of "sqlite3". This can be used + to run a specific sqlite3 version (e.g., one inside + a running container). If you run it from within + a container, make sure to mount your host's + ".borgmatic" folder into the container using the same + directory structure. Defaults to "sqlite3". + example: docker exec sqlite_container sqlite3 + sqlite_restore_command: + type: string + description: | + Command to run when restoring a database instead + of "sqlite3". This can be used to run a specific + sqlite3 version (e.g., one inside a running container). + Defaults to "sqlite3". + example: docker exec sqlite_container sqlite3 mongodb_databases: type: array items: diff --git a/borgmatic/hooks/data_source/sqlite.py b/borgmatic/hooks/data_source/sqlite.py index 8c4b6008..c7ed267b 100644 --- a/borgmatic/hooks/data_source/sqlite.py +++ b/borgmatic/hooks/data_source/sqlite.py @@ -79,13 +79,17 @@ def dump_data_sources( ) continue - command = ( - 'sqlite3', + sqlite_command = tuple( + shlex.quote(part) + for part in shlex.split(database.get('sqlite_command') or 'sqlite3') + ) + command = sqlite_command + ( shlex.quote(database_path), '.dump', '>', shlex.quote(dump_filename), ) + logger.debug( f'Dumping SQLite database at {database_path} to {dump_filename}{dry_run_label}' ) @@ -168,11 +172,11 @@ def restore_data_source_dump( except FileNotFoundError: # pragma: no cover pass - restore_command = ( - 'sqlite3', - database_path, + sqlite_restore_command = tuple( + shlex.quote(part) + for part in shlex.split(data_source.get('sqlite_restore_command') or 'sqlite3') ) - + restore_command = sqlite_restore_command + (shlex.quote(database_path),) # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning # if the restore paths don't exist in the archive. execute_command_with_processes( diff --git a/tests/unit/hooks/data_source/test_sqlite.py b/tests/unit/hooks/data_source/test_sqlite.py index 6c5a5639..b54af4ef 100644 --- a/tests/unit/hooks/data_source/test_sqlite.py +++ b/tests/unit/hooks/data_source/test_sqlite.py @@ -116,6 +116,51 @@ def test_dump_data_sources_with_path_injection_attack_gets_escaped(): ) +def test_dump_data_sources_runs_non_default_sqlite_with_path_injection_attack_gets_escaped(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) + databases = [ + { + 'path': '/path/to/database1; naughty-command', + 'name': 'database1', + 'sqlite_command': 'custom_sqlite *', + }, + ] + processes = [flexmock()] + + flexmock(module).should_receive('make_dump_path').and_return('/run/borgmatic') + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( + '/run/borgmatic/database' + ) + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + flexmock(module).should_receive('execute_command').with_args( + ( + 'custom_sqlite', # custom sqlite command + "'*'", # Should get shell escaped to prevent injection attacks. + "'/path/to/database1; naughty-command'", + '.dump', + '>', + '/run/borgmatic/database', + ), + shell=True, + run_to_completion=False, + ).and_return(processes[0]) + + assert ( + module.dump_data_sources( + databases, + {}, + config_paths=('test.yaml',), + borgmatic_runtime_directory='/run/borgmatic', + patterns=[], + dry_run=False, + ) + == processes + ) + + def test_dump_data_sources_with_non_existent_path_warns_and_dumps_database(): flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( flexmock() @@ -234,6 +279,41 @@ def test_restore_data_source_dump_restores_database(): ) +def test_restore_data_source_dump_runs_non_default_sqlite_restores_database(): + hook_config = [ + { + 'path': '/path/to/database', + 'name': 'database', + 'sqlite_restore_command': 'custom_sqlite *', + }, + {'name': 'other'}, + ] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'custom_sqlite', + "'*'", # Should get shell escaped to prevent injection attacks. + '/path/to/database', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + ).once() + + flexmock(module.os).should_receive('remove').once() + + module.restore_data_source_dump( + hook_config, + {}, + data_source=hook_config[0], + dry_run=False, + extract_process=extract_process, + connection_params={'restore_path': None}, + borgmatic_runtime_directory='/run/borgmatic', + ) + + def test_restore_data_source_dump_with_connection_params_uses_connection_params_for_restore(): hook_config = [ {'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'} @@ -263,6 +343,38 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ ) +def test_restore_data_source_dump_runs_non_default_sqlite_with_connection_params_uses_connection_params_for_restore(): + hook_config = [ + {'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'} + ] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'custom_sqlite', + 'cli/path/to/database', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + ).once() + + flexmock(module.os).should_receive('remove').once() + + module.restore_data_source_dump( + hook_config, + {}, + data_source={ + 'name': 'database', + 'sqlite_restore_command': 'custom_sqlite', + }, + dry_run=False, + extract_process=extract_process, + connection_params={'restore_path': 'cli/path/to/database'}, + borgmatic_runtime_directory='/run/borgmatic', + ) + + def test_restore_data_source_dump_without_connection_params_uses_restore_params_in_config_for_restore(): hook_config = [ {'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'} @@ -292,6 +404,40 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ ) +def test_restore_data_source_dump_runs_non_default_sqlite_without_connection_params_uses_restore_params_in_config_for_restore(): + hook_config = [ + { + 'path': '/path/to/database', + 'name': 'database', + 'sqlite_restore_command': 'custom_sqlite', + 'restore_path': 'config/path/to/database', + } + ] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'custom_sqlite', + 'config/path/to/database', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + ).once() + + flexmock(module.os).should_receive('remove').once() + + module.restore_data_source_dump( + hook_config, + {}, + data_source=hook_config[0], + dry_run=False, + extract_process=extract_process, + connection_params={'restore_path': None}, + borgmatic_runtime_directory='/run/borgmatic', + ) + + def test_restore_data_source_dump_does_not_restore_database_if_dry_run(): hook_config = [{'path': '/path/to/database', 'name': 'database'}] extract_process = flexmock(stdout=flexmock()) From f9612cc685b47748010eec787bf0bb7434b09467 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 15 Mar 2025 21:37:23 -0700 Subject: [PATCH 112/226] Add SQLite custom command option to NEWS (#836). --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index b3d053fb..ef18be2b 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,7 @@ * #790, #821: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more flexible "commands:". See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ + * #836: Add a custom command option for the SQLite hook. * #1010: When using Borg 2, don't pass the "--stats" flag to "borg prune". * #1020: Document a database use case involving a temporary database client container: https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers From 1d5713c4c5b645d9dd69ce4c41ada6949852b7b8 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 15 Mar 2025 21:42:45 -0700 Subject: [PATCH 113/226] Updated outdated schema comment referencing ~/.borgmatic path (#836). --- borgmatic/config/schema.yaml | 43 ++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index dbd79d0c..61c0c5e9 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1267,11 +1267,11 @@ properties: Command to use instead of "pg_dump" or "pg_dumpall". This can be used to run a specific pg_dump version (e.g., one inside a running container). If you run it - from within a container, make sure to mount your - host's ".borgmatic" folder into the container using - the same directory structure. Defaults to "pg_dump" - for single database dump or "pg_dumpall" to dump all - databases. + from within a container, make sure to mount the path in + the "user_runtime_directory" option from the host into + the container at the same location. Defaults to + "pg_dump" for single database dump or "pg_dumpall" to + dump all databases. example: docker exec my_pg_container pg_dump pg_restore_command: type: string @@ -1408,10 +1408,11 @@ properties: description: | Command to use instead of "mariadb-dump". This can be used to run a specific mariadb_dump version (e.g., one - inside a running container). If you run it from within - a container, make sure to mount your host's - ".borgmatic" folder into the container using the same - directory structure. Defaults to "mariadb-dump". + inside a running container). If you run it from within a + container, make sure to mount the path in the + "user_runtime_directory" option from the host into the + container at the same location. Defaults to + "mariadb-dump". example: docker exec mariadb_container mariadb-dump mariadb_command: type: string @@ -1550,12 +1551,12 @@ properties: mysql_dump_command: type: string description: | - Command to use instead of "mysqldump". This can be - used to run a specific mysql_dump version (e.g., one - inside a running container). If you run it from within - a container, make sure to mount your host's - ".borgmatic" folder into the container using the same - directory structure. Defaults to "mysqldump". + Command to use instead of "mysqldump". This can be used + to run a specific mysql_dump version (e.g., one inside a + running container). If you run it from within a + container, make sure to mount the path in the + "user_runtime_directory" option from the host into the + container at the same location. Defaults to "mysqldump". example: docker exec mysql_container mysqldump mysql_command: type: string @@ -1645,12 +1646,12 @@ properties: sqlite_command: type: string description: | - Command to use instead of "sqlite3". This can be used - to run a specific sqlite3 version (e.g., one inside - a running container). If you run it from within - a container, make sure to mount your host's - ".borgmatic" folder into the container using the same - directory structure. Defaults to "sqlite3". + Command to use instead of "sqlite3". This can be used to + run a specific sqlite3 version (e.g., one inside a + running container). If you run it from within a + container, make sure to mount the path in the + "user_runtime_directory" option from the host into the + container at the same location. Defaults to "sqlite3". example: docker exec sqlite_container sqlite3 sqlite_restore_command: type: string From 05900c188f6f0426a0ac78371683a1af17030bc9 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 15 Mar 2025 22:58:39 -0700 Subject: [PATCH 114/226] Expand docstrings (#303). --- borgmatic/config/arguments.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/borgmatic/config/arguments.py b/borgmatic/config/arguments.py index 96a28b4d..ec7fe8ee 100644 --- a/borgmatic/config/arguments.py +++ b/borgmatic/config/arguments.py @@ -10,14 +10,25 @@ LIST_INDEX_KEY_PATTERN = re.compile(r'^(?P[a-zA-z-]+)\[(?P\d+) def set_values(config, keys, value): ''' Given a configuration dict, a sequence of parsed key strings, and a string value, descend into - the configuration hierarchy based on the keys to set the value into the right place. + the configuration hierarchy based on the given keys and set the value into the right place. + For example, consider these keys: + + ('foo', 'bar', 'baz') + + This looks up "foo" in the given configuration. And within that value, it looks up "bar". And + then within that value, it looks up "baz" and sets it to the given value. Another example: + + ('mylist[0]', 'foo') + + This looks for the zeroth element of "mylist" in the given configuration. And within that value, + it looks up "foo" and sets it to the given value. Finally: ''' if not keys: return first_key = keys[0] - # Support "name[0]"-style list index syntax. + # Support "mylist[0]" list index syntax. match = LIST_INDEX_KEY_PATTERN.match(first_key) if match: @@ -41,6 +52,7 @@ def set_values(config, keys, value): if len(keys) == 1: config[first_key] = value + return if first_key not in config: From 6adb0fd44cc7be8d7dded5f39ee005622607ff6d Mon Sep 17 00:00:00 2001 From: Vandal <86826719+VandalByte@users.noreply.github.com> Date: Mon, 17 Mar 2025 22:24:53 +0530 Subject: [PATCH 115/226] add borg recreate --- borgmatic/actions/recreate.py | 0 borgmatic/borg/recreate.py | 82 +++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 borgmatic/actions/recreate.py create mode 100644 borgmatic/borg/recreate.py diff --git a/borgmatic/actions/recreate.py b/borgmatic/actions/recreate.py new file mode 100644 index 00000000..e69de29b diff --git a/borgmatic/borg/recreate.py b/borgmatic/borg/recreate.py new file mode 100644 index 00000000..41cd4eef --- /dev/null +++ b/borgmatic/borg/recreate.py @@ -0,0 +1,82 @@ +import argparse +import logging + +import borgmatic.borg.environment +import borgmatic.borg.feature +import borgmatic.borg.flags +import borgmatic.borg.repo_delete +import borgmatic.config.paths +import borgmatic.execute +from borgmatic.borg.create import make_exclude_flags, write_patterns_file + +logger = logging.getLogger(__name__) + + +def make_recreate_command( + repository, + config, + patterns, + local_borg_version, + recreate_arguments, + global_arguments, + local_path, + remote_path=None, +): + ''' + Given a repository path, configuration dict, patterns, and Borg options, return a command + list for recreating a Borg archive. + ''' + verbosity_flags = () + if logger.isEnabledFor(logging.DEBUG): + verbosity_flags = ('--debug', '--show-rc') + elif logger.isEnabledFor(logging.INFO): + verbosity_flags = ('--info',) + + command = [local_path, 'recreate', repository] + command.extend(verbosity_flags) + command.extend(global_arguments) + command.extend(recreate_arguments) + + exclude_flags = make_exclude_flags(config) + command.extend(exclude_flags) + + return command + + +def recreate_archive( + repository, + config, + patterns, + local_borg_version, + recreate_arguments, + global_arguments, + local_path='borg', + remote_path=None, +): + ''' + Recreate a Borg archive with the given repository and configuration. + ''' + command = make_recreate_command( + repository, + config, + patterns, + local_borg_version, + recreate_arguments, + global_arguments, + local_path, + remote_path, + ) + + patterns_file = write_patterns_file(patterns, borgmatic.config.paths.get_runtime_directory()) + if patterns_file: + command.extend(['--patterns-from', patterns_file.name]) + + borgmatic.execute.execute_command( + command, + output_log_level=logging.ANSWER, + environment=borgmatic.borg.environment.make_environment(config), + working_directory=borgmatic.config.paths.get_working_directory(config), + remote_path=remote_path, + borg_local_path=local_path, + borg_exit_codes=config.get('borg_exit_codes') + ) From eca78fbc2cdeade1c6e4a227ebbc018664468300 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 17 Mar 2025 09:57:25 -0700 Subject: [PATCH 116/226] Support setting whole lists and dicts from the command-line (#303). --- borgmatic/commands/arguments.py | 45 +++++++++++------ borgmatic/config/arguments.py | 22 ++++++-- borgmatic/config/generate.py | 29 ++--------- borgmatic/config/normalize.py | 6 ++- borgmatic/config/schema.py | 22 ++++++++ borgmatic/config/schema.yaml | 36 +++++++++++++- tests/unit/config/test_generate.py | 76 ---------------------------- tests/unit/config/test_schema.py | 80 ++++++++++++++++++++++++++++++ 8 files changed, 194 insertions(+), 122 deletions(-) create mode 100644 borgmatic/config/schema.py create mode 100644 tests/unit/config/test_schema.py diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 6504257b..555f41de 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1,11 +1,16 @@ import collections +import decimal import itertools +import io import json import re import sys from argparse import ArgumentParser from borgmatic.config import collect +import borgmatic.config.schema + +import ruamel.yaml ACTION_ALIASES = { 'repo-create': ['rcreate', 'init', '-I'], @@ -308,8 +313,9 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names --foo.bar - If "foo" is instead an array of objects, it will get added like this + If "foo" is instead an array of objects, both of the following will get added: + --foo --foo[0].bar And if names are also passed in, they are considered to be the name components of an option @@ -328,12 +334,8 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names raise ValueError(f'Unknown type in configuration schema: {schema_type}') # If this is an "object" type, recurse for each child option ("property"). - if schema_type in {'object', 'array'}: - properties = ( - schema.get('items', {}).get('properties') - if schema_type == 'array' - else schema.get('properties') - ) + if schema_type == 'object': + properties = schema.get('properties') if properties: for name, child in properties.items(): @@ -341,23 +343,37 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names arguments_group, child, unparsed_arguments, - names + ((name + '[0]',) if child.get('type') == 'array' else (name,)), + names + (name,) ) return + # If this is an "array" type, recurse for each child option of its items type. Don't return yet, + # so that a flag also gets added below for the array itself. + if schema_type == 'array': + properties = borgmatic.config.schema.get_properties(schema.get('items', {})) + + if properties: + for name, child in properties.items(): + add_arguments_from_schema( + arguments_group, + child, + unparsed_arguments, + names[:-1] + (f'{names[-1]}[0]',) + (name,) + ) + flag_name = '.'.join(names) description = schema.get('description') metavar = names[-1].upper() - if schema_type == 'array': - metavar = metavar.rstrip('S') - if description: if schema_type == 'array': - items_schema = schema.get('items', {}) + example_buffer = io.StringIO() + yaml = ruamel.yaml.YAML(typ='safe') + yaml.default_flow_style = True + yaml.dump(schema.get('example'), example_buffer) - description += ' Can specify flag multiple times.' + description += f' Example value: "{example_buffer.getvalue().strip()}"' if '[0]' in flag_name: description += ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).' @@ -365,7 +381,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names description = description.replace('%', '%%') try: - argument_type = {'string': str, 'integer': int, 'boolean': bool, 'array': str}[schema_type] + argument_type = {'string': str, 'integer': int, 'number': decimal.Decimal, 'boolean': bool, 'array': str}[schema_type] except KeyError: raise ValueError(f'Unknown type in configuration schema: {schema_type}') @@ -373,7 +389,6 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names f"--{flag_name.replace('_', '-')}", type=argument_type, metavar=metavar, - action='append' if schema_type == 'array' else None, help=description, ) diff --git a/borgmatic/config/arguments.py b/borgmatic/config/arguments.py index ec7fe8ee..dab37b5b 100644 --- a/borgmatic/config/arguments.py +++ b/borgmatic/config/arguments.py @@ -21,7 +21,7 @@ def set_values(config, keys, value): ('mylist[0]', 'foo') This looks for the zeroth element of "mylist" in the given configuration. And within that value, - it looks up "foo" and sets it to the given value. Finally: + it looks up "foo" and sets it to the given value. ''' if not keys: return @@ -89,6 +89,22 @@ def type_for_option(schema, option_keys): return None +def convert_value_type(value, option_type): + ''' + Given a string value and its schema type as a string, determine its logical type (string, + boolean, integer, etc.), and return it converted to that type. + + If the option type is a string, leave the value as a string so that special characters in it + don't get interpreted as YAML during conversion. + + Raise ruamel.yaml.error.YAMLError if there's a parse issue with the YAML. + ''' + if option_type == 'string': + return value + + return ruamel.yaml.YAML(typ='safe').load(io.StringIO(value)) + + def prepare_arguments_for_config(global_arguments, schema): ''' Given global arguments as an argparse.Namespace and a configuration schema dict, parse each @@ -125,11 +141,11 @@ def prepare_arguments_for_config(global_arguments, schema): prepared_values.append( ( keys, - value, + convert_value_type(value, option_type), ) ) except ruamel.yaml.error.YAMLError as error: - raise ValueError(f"Invalid override '{raw_override}': {error.problem}") + raise ValueError(f'Invalid override "{argument_name}": {error.problem}') return tuple(prepared_values) diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index 58fd03a7..e0a646d8 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -1,12 +1,12 @@ import collections import io -import itertools import os import re import ruamel.yaml from borgmatic.config import load, normalize +import borgmatic.config.schema INDENT = 4 SEQUENCE_INDENT = 2 @@ -22,27 +22,6 @@ def insert_newline_before_comment(config, field_name): ) -def get_properties(schema): - ''' - Given a schema dict, return its properties. But if it's got sub-schemas with multiple different - potential properties, returned their merged properties instead (interleaved so the first - properties of each sub-schema come first). The idea is that the user should see all possible - options even if they're not all possible together. - ''' - if 'oneOf' in schema: - return dict( - item - for item in itertools.chain( - *itertools.zip_longest( - *[sub_schema['properties'].items() for sub_schema in schema['oneOf']] - ) - ) - if item is not None - ) - - return schema['properties'] - - def schema_to_sample_configuration(schema, source_config=None, level=0, parent_is_sequence=False): ''' Given a loaded configuration schema and a source configuration, generate and return sample @@ -78,7 +57,7 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i sub_schema, (source_config or {}).get(field_name, {}), level + 1 ), ) - for field_name, sub_schema in get_properties(schema).items() + for field_name, sub_schema in borgmatic.config.schema.get_properties(schema).items() ] ) indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0) @@ -189,7 +168,7 @@ def add_comments_to_configuration_sequence(config, schema, indent=0): return for field_name in config[0].keys(): - field_schema = get_properties(schema['items']).get(field_name, {}) + field_schema = borgmatic.config.schema.get_properties(schema['items']).get(field_name, {}) description = field_schema.get('description') # No description to use? Skip it. @@ -223,7 +202,7 @@ def add_comments_to_configuration_object( if skip_first and index == 0: continue - field_schema = get_properties(schema).get(field_name, {}) + field_schema = borgmatic.config.schema.get_properties(schema).get(field_name, {}) description = field_schema.get('description', '').strip() # If this isn't a default key, add an indicator to the comment flagging it to be commented diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index 11f21ce0..f4199e6b 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -326,7 +326,11 @@ def normalize(config_filename, config): config['repositories'] = [] for repository_dict in repositories: - repository_path = repository_dict['path'] + repository_path = repository_dict.get('path') + + if repository_path is None: + continue + if '~' in repository_path: logs.append( logging.makeLogRecord( diff --git a/borgmatic/config/schema.py b/borgmatic/config/schema.py new file mode 100644 index 00000000..e46d27cc --- /dev/null +++ b/borgmatic/config/schema.py @@ -0,0 +1,22 @@ +import itertools + + +def get_properties(schema): + ''' + Given a schema dict, return its properties. But if it's got sub-schemas with multiple different + potential properties, returned their merged properties instead (interleaved so the first + properties of each sub-schema come first). The idea is that the user should see all possible + options even if they're not all possible together. + ''' + if 'oneOf' in schema: + return dict( + item + for item in itertools.chain( + *itertools.zip_longest( + *[sub_schema['properties'].items() for sub_schema in schema['oneOf']] + ) + ) + if item is not None + ) + + return schema.get('properties', {}) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 3c1bb20f..2f892337 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -53,8 +53,7 @@ properties: output of "borg help placeholders" for details. See ssh_command for SSH options like identity file or port. If systemd service is used, then add local repository paths in the systemd service file to the - ReadWritePaths list. Prior to borgmatic 1.7.10, repositories was a - list of plain path strings. + ReadWritePaths list. example: - path: ssh://user@backupserver/./sourcehostname.borg label: backupserver @@ -738,6 +737,10 @@ properties: List of one or more consistency checks to run on a periodic basis (if "frequency" is set) or every time borgmatic runs checks (if "frequency" is omitted). + example: + - name: archives + frequency: 2 weeks + - name: repository check_repositories: type: array items: @@ -1115,6 +1118,10 @@ properties: List of one or more command hooks to execute, triggered at particular points during borgmatic's execution. For each command hook, specify one of "before" or "after", not both. + example: + - before: action + when: [create] + run: [echo Backing up.] bootstrap: type: object properties: @@ -1329,6 +1336,9 @@ properties: https://www.postgresql.org/docs/current/app-pgdump.html and https://www.postgresql.org/docs/current/libpq-ssl.html for details. + example: + - name: users + hostname: database.example.org mariadb_databases: type: array items: @@ -1473,6 +1483,9 @@ properties: added to your source directories at runtime and streamed directly to Borg. Requires mariadb-dump/mariadb commands. See https://mariadb.com/kb/en/library/mysqldump/ for details. + example: + - name: users + hostname: database.example.org mysql_databases: type: array items: @@ -1618,6 +1631,9 @@ properties: to Borg. Requires mysqldump/mysql commands. See https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html for details. + example: + - name: users + hostname: database.example.org sqlite_databases: type: array items: @@ -1647,6 +1663,15 @@ properties: Path to the SQLite database file to restore to. Defaults to the "path" option. example: /var/lib/sqlite/users.db + description: | + List of one or more SQLite databases to dump before creating a + backup, run once per configuration file. The database dumps are + added to your source directories at runtime and streamed directly to + Borg. Requires the sqlite3 command. See https://sqlite.org/cli.html + for details. + example: + - name: users + path: /var/lib/db.sqlite mongodb_databases: type: array items: @@ -1749,6 +1774,9 @@ properties: to Borg. Requires mongodump/mongorestore commands. See https://docs.mongodb.com/database-tools/mongodump/ and https://docs.mongodb.com/database-tools/mongorestore/ for details. + example: + - name: users + hostname: database.example.org ntfy: type: object required: ['topic'] @@ -2231,9 +2259,13 @@ properties: properties: url: type: string + description: URL of this Apprise service. example: "gotify://hostname/token" label: type: string + description: | + Label used in borgmatic logs for this Apprise + service. example: gotify description: | A list of Apprise services to publish to with URLs and diff --git a/tests/unit/config/test_generate.py b/tests/unit/config/test_generate.py index 27dcd01f..abb520fb 100644 --- a/tests/unit/config/test_generate.py +++ b/tests/unit/config/test_generate.py @@ -4,82 +4,6 @@ from flexmock import flexmock from borgmatic.config import generate as module -def test_get_properties_with_simple_object(): - schema = { - 'type': 'object', - 'properties': dict( - [ - ('field1', {'example': 'Example'}), - ] - ), - } - - assert module.get_properties(schema) == schema['properties'] - - -def test_get_properties_merges_oneof_list_properties(): - schema = { - 'type': 'object', - 'oneOf': [ - { - 'properties': dict( - [ - ('field1', {'example': 'Example 1'}), - ('field2', {'example': 'Example 2'}), - ] - ), - }, - { - 'properties': dict( - [ - ('field2', {'example': 'Example 2'}), - ('field3', {'example': 'Example 3'}), - ] - ), - }, - ], - } - - assert module.get_properties(schema) == dict( - schema['oneOf'][0]['properties'], **schema['oneOf'][1]['properties'] - ) - - -def test_get_properties_interleaves_oneof_list_properties(): - schema = { - 'type': 'object', - 'oneOf': [ - { - 'properties': dict( - [ - ('field1', {'example': 'Example 1'}), - ('field2', {'example': 'Example 2'}), - ('field3', {'example': 'Example 3'}), - ] - ), - }, - { - 'properties': dict( - [ - ('field4', {'example': 'Example 4'}), - ('field5', {'example': 'Example 5'}), - ] - ), - }, - ], - } - - assert module.get_properties(schema) == dict( - [ - ('field1', {'example': 'Example 1'}), - ('field4', {'example': 'Example 4'}), - ('field2', {'example': 'Example 2'}), - ('field5', {'example': 'Example 5'}), - ('field3', {'example': 'Example 3'}), - ] - ) - - def test_schema_to_sample_configuration_generates_config_map_with_examples(): schema = { 'type': 'object', diff --git a/tests/unit/config/test_schema.py b/tests/unit/config/test_schema.py new file mode 100644 index 00000000..e10992b6 --- /dev/null +++ b/tests/unit/config/test_schema.py @@ -0,0 +1,80 @@ +from borgmatic.config import schema as module + + +def test_get_properties_with_simple_object(): + schema = { + 'type': 'object', + 'properties': dict( + [ + ('field1', {'example': 'Example'}), + ] + ), + } + + assert module.get_properties(schema) == schema['properties'] + + +def test_get_properties_merges_oneof_list_properties(): + schema = { + 'type': 'object', + 'oneOf': [ + { + 'properties': dict( + [ + ('field1', {'example': 'Example 1'}), + ('field2', {'example': 'Example 2'}), + ] + ), + }, + { + 'properties': dict( + [ + ('field2', {'example': 'Example 2'}), + ('field3', {'example': 'Example 3'}), + ] + ), + }, + ], + } + + assert module.get_properties(schema) == dict( + schema['oneOf'][0]['properties'], **schema['oneOf'][1]['properties'] + ) + + +def test_get_properties_interleaves_oneof_list_properties(): + schema = { + 'type': 'object', + 'oneOf': [ + { + 'properties': dict( + [ + ('field1', {'example': 'Example 1'}), + ('field2', {'example': 'Example 2'}), + ('field3', {'example': 'Example 3'}), + ] + ), + }, + { + 'properties': dict( + [ + ('field4', {'example': 'Example 4'}), + ('field5', {'example': 'Example 5'}), + ] + ), + }, + ], + } + + assert module.get_properties(schema) == dict( + [ + ('field1', {'example': 'Example 1'}), + ('field4', {'example': 'Example 4'}), + ('field2', {'example': 'Example 2'}), + ('field5', {'example': 'Example 5'}), + ('field3', {'example': 'Example 3'}), + ] + ) + + + From 87b9ad5aeaebdf02b4d448b97a5ac9ecf221dc6d Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 17 Mar 2025 10:02:25 -0700 Subject: [PATCH 117/226] Code formatting (#303). --- borgmatic/commands/arguments.py | 23 +++++++++++++---------- borgmatic/config/arguments.py | 1 - borgmatic/config/generate.py | 2 +- borgmatic/config/override.py | 5 +++-- borgmatic/config/validate.py | 4 +++- tests/unit/config/test_schema.py | 3 --- 6 files changed, 20 insertions(+), 18 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 555f41de..6b3abf9c 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1,17 +1,17 @@ import collections import decimal -import itertools import io +import itertools import json import re import sys from argparse import ArgumentParser -from borgmatic.config import collect -import borgmatic.config.schema - import ruamel.yaml +import borgmatic.config.schema +from borgmatic.config import collect + ACTION_ALIASES = { 'repo-create': ['rcreate', 'init', '-I'], 'prune': ['-p'], @@ -340,10 +340,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names if properties: for name, child in properties.items(): add_arguments_from_schema( - arguments_group, - child, - unparsed_arguments, - names + (name,) + arguments_group, child, unparsed_arguments, names + (name,) ) return @@ -359,7 +356,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names arguments_group, child, unparsed_arguments, - names[:-1] + (f'{names[-1]}[0]',) + (name,) + names[:-1] + (f'{names[-1]}[0]',) + (name,), ) flag_name = '.'.join(names) @@ -381,7 +378,13 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names description = description.replace('%', '%%') try: - argument_type = {'string': str, 'integer': int, 'number': decimal.Decimal, 'boolean': bool, 'array': str}[schema_type] + argument_type = { + 'string': str, + 'integer': int, + 'number': decimal.Decimal, + 'boolean': bool, + 'array': str, + }[schema_type] except KeyError: raise ValueError(f'Unknown type in configuration schema: {schema_type}') diff --git a/borgmatic/config/arguments.py b/borgmatic/config/arguments.py index dab37b5b..7a71c926 100644 --- a/borgmatic/config/arguments.py +++ b/borgmatic/config/arguments.py @@ -3,7 +3,6 @@ import re import ruamel.yaml - LIST_INDEX_KEY_PATTERN = re.compile(r'^(?P[a-zA-z-]+)\[(?P\d+)\]$') diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index e0a646d8..fbc29c81 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -5,8 +5,8 @@ import re import ruamel.yaml -from borgmatic.config import load, normalize import borgmatic.config.schema +from borgmatic.config import load, normalize INDENT = 4 SEQUENCE_INDENT = 2 diff --git a/borgmatic/config/override.py b/borgmatic/config/override.py index 0893f669..9067e6f9 100644 --- a/borgmatic/config/override.py +++ b/borgmatic/config/override.py @@ -3,7 +3,6 @@ import logging import ruamel.yaml - logger = logging.getLogger(__name__) @@ -139,7 +138,9 @@ def apply_overrides(config, schema, raw_overrides): overrides = parse_overrides(raw_overrides, schema) if overrides: - logger.warning("The --override flag is deprecated and will be removed from a future release. Instead, use a command-line flag corresponding to the configuration option you'd like to set.") + logger.warning( + "The --override flag is deprecated and will be removed from a future release. Instead, use a command-line flag corresponding to the configuration option you'd like to set." + ) for keys, value in overrides: set_values(config, keys, value) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 35c8e809..063f5767 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -84,7 +84,9 @@ def apply_logical_validation(config_filename, parsed_configuration): ) -def parse_configuration(config_filename, schema_filename, global_arguments, overrides=None, resolve_env=True): +def parse_configuration( + config_filename, schema_filename, global_arguments, overrides=None, resolve_env=True +): ''' Given the path to a config filename in YAML format, the path to a schema filename in a YAML rendition of JSON Schema format, global arguments as an argparse.Namespace, a sequence of diff --git a/tests/unit/config/test_schema.py b/tests/unit/config/test_schema.py index e10992b6..a07b19ba 100644 --- a/tests/unit/config/test_schema.py +++ b/tests/unit/config/test_schema.py @@ -75,6 +75,3 @@ def test_get_properties_interleaves_oneof_list_properties(): ('field3', {'example': 'Example 3'}), ] ) - - - From 7b14e8c7f2d1264ae75379181d52a14fb9e9f13f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 17 Mar 2025 10:17:04 -0700 Subject: [PATCH 118/226] Add feature to NEWS (#303). --- NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS b/NEWS index b3d053fb..60e74e22 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,7 @@ 2.0.0.dev0 + * #303: Add flags for setting any borgmatic configuration option from the command-line. See the + documentation for more information: + https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides * #790, #821: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more flexible "commands:". See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ From c926f0bd5d75095ac4b70852279c38b68d9a7d9c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 17 Mar 2025 10:31:34 -0700 Subject: [PATCH 119/226] Clarify documentation for `dump_data_sources` command hook (#790). --- docs/how-to/add-preparation-and-cleanup-steps-to-backups.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index 578601e2..8001a4c3 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -78,7 +78,8 @@ commands: This command hook has the following options: - * `before` or `after`: `dump_data_sources` + * `before` or `after`: Name for the point in borgmatic's execution that the commands should be run before or after: + * `dump_data_sources` runs before or after data sources are dumped (databases dumped or filesystems snapshotted) for each hook named in `hooks`. * `hooks`: Names of other hooks that this command hook applies to, e.g. `postgresql`, `mariadb`, `zfs`, `btrfs`, etc. Defaults to all hooks of the relevant type. * `run`: One or more shell commands or scripts to run when this command hook is triggered. From 903308864cbf8622c308118ca2c3170504038d4e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 17 Mar 2025 10:46:02 -0700 Subject: [PATCH 120/226] Factor out schema type parsing (#303). --- borgmatic/commands/arguments.py | 12 +----------- borgmatic/config/schema.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 6b3abf9c..dad5584d 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1,5 +1,4 @@ import collections -import decimal import io import itertools import json @@ -377,16 +376,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names description = description.replace('%', '%%') - try: - argument_type = { - 'string': str, - 'integer': int, - 'number': decimal.Decimal, - 'boolean': bool, - 'array': str, - }[schema_type] - except KeyError: - raise ValueError(f'Unknown type in configuration schema: {schema_type}') + argument_type = borgmatic.config.schema.parse_type(schema_type) arguments_group.add_argument( f"--{flag_name.replace('_', '-')}", diff --git a/borgmatic/config/schema.py b/borgmatic/config/schema.py index e46d27cc..118e9437 100644 --- a/borgmatic/config/schema.py +++ b/borgmatic/config/schema.py @@ -1,3 +1,4 @@ +import decimal import itertools @@ -20,3 +21,16 @@ def get_properties(schema): ) return schema.get('properties', {}) + + +def parse_type(schema_type): + try: + return { + 'string': str, + 'integer': int, + 'number': decimal.Decimal, + 'boolean': bool, + 'array': str, + }[schema_type] + except KeyError: + raise ValueError(f'Unknown type in configuration schema: {schema_type}') From 93e7da823caa9d7fd97d8253af9953547e1d9bf0 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 17 Mar 2025 22:24:01 -0700 Subject: [PATCH 121/226] Add an encryption option to repositories (#303). --- NEWS | 4 ++++ borgmatic/actions/repo_create.py | 10 +++++++++- borgmatic/commands/arguments.py | 6 ++++-- borgmatic/config/schema.yaml | 8 ++++++++ 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 4a1357a2..06f59533 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,10 @@ * #303: Add flags for setting any borgmatic configuration option from the command-line. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides + * #303: Add configuration options that serve as defaults for some (but not all) borgmatic + action flags. For example, each entry in "repositories:" now has an "encryption" option that + applies to the "repo-create" action. See the documentation for more information: + https://torsion.org/borgmatic/docs/reference/configuration/ * #790, #821: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more flexible "commands:". See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ diff --git a/borgmatic/actions/repo_create.py b/borgmatic/actions/repo_create.py index e6252e4e..af7b793d 100644 --- a/borgmatic/actions/repo_create.py +++ b/borgmatic/actions/repo_create.py @@ -24,13 +24,21 @@ def run_repo_create( return logger.info('Creating repository') + + encryption_mode = repo_create_arguments.encryption_mode or repository.get('encryption') + + if not encryption_mode: + raise ValueError( + 'With the repo-create action, either the --encryption flag or the repository encryption option is required.' + ) + borgmatic.borg.repo_create.create_repository( global_arguments.dry_run, repository['path'], config, local_borg_version, global_arguments, - repo_create_arguments.encryption_mode, + encryption_mode, repo_create_arguments.source_repository, repo_create_arguments.copy_crypt_key, repo_create_arguments.append_only, diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index dad5584d..3f2da08b 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -445,7 +445,10 @@ def make_parsers(schema, unparsed_arguments): config_paths = collect.get_default_config_paths(expand_home=True) unexpanded_config_paths = collect.get_default_config_paths(expand_home=False) - global_parser = ArgumentParser(add_help=False) + # allow_abbrev=False prevents the global parser from erroring about "ambiguous" options like + # --encryption. Such options are intended for an action parser rather than the global parser, + # and so we don't want to error on them here. + global_parser = ArgumentParser(allow_abbrev=False, add_help=False) global_group = global_parser.add_argument_group('global arguments') global_group.add_argument( @@ -569,7 +572,6 @@ def make_parsers(schema, unparsed_arguments): '--encryption', dest='encryption_mode', help='Borg repository encryption mode', - required=True, ) repo_create_group.add_argument( '--source-repository', diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 7fe30daf..e4901dc7 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -45,6 +45,14 @@ properties: and to make selecting the repository easier on the command-line. example: backupserver + encryption: + type: string + description: | + The encryption mode with which to create the repository, + only used for the repo-create action. To see the + available encryption modes, run "borg init --help" with + Borg 1 or "borg repo-create --help" with Borg 2. + example: repokey-blake2 description: | A required list of local or remote repositories with paths and optional labels (which can be used with the --repository flag to From 711f5fa6cb9c0d7b8e5626632b2d9d279ab3aa66 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 17 Mar 2025 22:58:25 -0700 Subject: [PATCH 122/226] UX nicety to make default-false boolean options into valueless CLI flags (#303). --- borgmatic/commands/arguments.py | 38 +++++++++++++++++++++++++++------ borgmatic/config/schema.yaml | 12 +++++------ 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 3f2da08b..72980327 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -288,6 +288,24 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse ) +# As a UX nicety, allow boolean options that have a default of false to have command-line flags +# without values. +DEFAULT_FALSE_FLAG_NAMES = { + 'one_file_system', + 'numeric_ids', + 'read_special', + 'exclude_caches', + 'keep_exclude_tags', + 'exclude_nodump', + 'source_directories_must_exist', + 'relocated_repo_access_is_ok', + 'unknown_unencrypted_repo_access_is_ok', + 'check_i_know_what_i_am_doing', + 'postgresql_databases[0].no_owner', + 'healthchecks.create_slug', +} + + def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names=None): ''' Given an argparse._ArgumentGroup instance, a configuration schema dict, and a sequence of @@ -377,13 +395,21 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names description = description.replace('%', '%%') argument_type = borgmatic.config.schema.parse_type(schema_type) + full_flag_name = f"--{flag_name.replace('_', '-')}" - arguments_group.add_argument( - f"--{flag_name.replace('_', '-')}", - type=argument_type, - metavar=metavar, - help=description, - ) + if flag_name in DEFAULT_FALSE_FLAG_NAMES: + arguments_group.add_argument( + full_flag_name, + action='store_true', + help=description, + ) + else: + arguments_group.add_argument( + full_flag_name, + type=argument_type, + metavar=metavar, + help=description, + ) # We want to support flags that can have arbitrary indices like: # diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index e4901dc7..909bc511 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1983,8 +1983,8 @@ properties: type: boolean description: | Set to True to enable HTML parsing of the message. - Set to False for plain text. - example: True + Set to false for plain text. + example: true sound: type: string description: | @@ -2058,8 +2058,8 @@ properties: type: boolean description: | Set to True to enable HTML parsing of the message. - Set to False for plain text. - example: True + Set to false for plain text. + example: true sound: type: string description: | @@ -2133,8 +2133,8 @@ properties: type: boolean description: | Set to True to enable HTML parsing of the message. - Set to False for plain text. - example: True + Set to false for plain text. + example: true sound: type: string description: | From 4e2805918d84991ceca661a9d873def9e977951c Mon Sep 17 00:00:00 2001 From: Vandal <86826719+VandalByte@users.noreply.github.com> Date: Tue, 18 Mar 2025 23:19:33 +0530 Subject: [PATCH 123/226] update borg/recreate.py --- borgmatic/borg/recreate.py | 64 +++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/borgmatic/borg/recreate.py b/borgmatic/borg/recreate.py index 41cd4eef..a9c6ff56 100644 --- a/borgmatic/borg/recreate.py +++ b/borgmatic/borg/recreate.py @@ -1,21 +1,18 @@ -import argparse import logging import borgmatic.borg.environment -import borgmatic.borg.feature -import borgmatic.borg.flags -import borgmatic.borg.repo_delete import borgmatic.config.paths import borgmatic.execute -from borgmatic.borg.create import make_exclude_flags, write_patterns_file +from borgmatic.borg.create import make_exclude_flags +from borgmatic.borg.flags import make_flags_from_arguments, make_repository_archive_flags logger = logging.getLogger(__name__) def make_recreate_command( repository, + archive, config, - patterns, local_borg_version, recreate_arguments, global_arguments, @@ -23,30 +20,39 @@ def make_recreate_command( remote_path=None, ): ''' - Given a repository path, configuration dict, patterns, and Borg options, return a command - list for recreating a Borg archive. + Given a local or remote repository path, an archive name, a configuration dict, + the local Borg version string, an argparse.Namespace of recreate arguments, + an argparse.Namespace of global arguments, optional local and remote Borg paths. + + Returns the recreate command as a tuple of strings ready for execution. ''' - verbosity_flags = () - if logger.isEnabledFor(logging.DEBUG): - verbosity_flags = ('--debug', '--show-rc') - elif logger.isEnabledFor(logging.INFO): - verbosity_flags = ('--info',) + verbosity_flags = (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + ( + ('--info',) if logger.isEnabledFor(logging.INFO) else () + ) - command = [local_path, 'recreate', repository] - command.extend(verbosity_flags) - command.extend(global_arguments) - command.extend(recreate_arguments) + # handle both the recreate and global arguments + recreate_flags = make_flags_from_arguments( + recreate_arguments, excludes=('repository', 'archive') + ) + global_flags = make_flags_from_arguments(global_arguments) + repo_archive_flags = make_repository_archive_flags(repository, archive, local_borg_version) exclude_flags = make_exclude_flags(config) - command.extend(exclude_flags) - return command + return ( + (local_path, 'recreate') + + repo_archive_flags + + verbosity_flags + + global_flags + + recreate_flags + + exclude_flags + ) def recreate_archive( repository, + archive, config, - patterns, local_borg_version, recreate_arguments, global_arguments, @@ -54,12 +60,16 @@ def recreate_archive( remote_path=None, ): ''' - Recreate a Borg archive with the given repository and configuration. + Given a local or remote repository path, an archive name, a configuration dict, + the local Borg version string, an argparse.Namespace of recreate arguments, + an argparse.Namespace of global arguments, optional local and remote Borg paths. + + Executes the recreate command with the given arguments. ''' command = make_recreate_command( repository, + archive, config, - patterns, local_borg_version, recreate_arguments, global_arguments, @@ -67,16 +77,12 @@ def recreate_archive( remote_path, ) - patterns_file = write_patterns_file(patterns, borgmatic.config.paths.get_runtime_directory()) - if patterns_file: - command.extend(['--patterns-from', patterns_file.name]) - borgmatic.execute.execute_command( command, output_log_level=logging.ANSWER, environment=borgmatic.borg.environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), - remote_path=remote_path, + remote_path=remote_path, borg_local_path=local_path, - borg_exit_codes=config.get('borg_exit_codes') - ) + borg_exit_codes=config.get('borg_exit_codes'), + ) From 63b0c69794a1fb4f36be0b66811db4d4ef02c15f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 18 Mar 2025 20:54:14 -0700 Subject: [PATCH 124/226] Add additional options under "repositories:" for parity with repo-create #303. --- borgmatic/actions/repo_create.py | 6 +++--- borgmatic/commands/arguments.py | 3 +++ borgmatic/config/arguments.py | 9 +++++++-- borgmatic/config/generate.py | 12 ++++++++---- borgmatic/config/schema.yaml | 20 ++++++++++++++++++++ 5 files changed, 41 insertions(+), 9 deletions(-) diff --git a/borgmatic/actions/repo_create.py b/borgmatic/actions/repo_create.py index af7b793d..49f4e6c0 100644 --- a/borgmatic/actions/repo_create.py +++ b/borgmatic/actions/repo_create.py @@ -41,9 +41,9 @@ def run_repo_create( encryption_mode, repo_create_arguments.source_repository, repo_create_arguments.copy_crypt_key, - repo_create_arguments.append_only, - repo_create_arguments.storage_quota, - repo_create_arguments.make_parent_dirs, + repo_create_arguments.append_only or repository.get('append_only'), + repo_create_arguments.storage_quota or repository.get('storage_quota'), + repo_create_arguments.make_parent_dirs or repository.get('make_parent_dirs'), local_path=local_path, remote_path=remote_path, ) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 72980327..bf65c24b 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -303,6 +303,8 @@ DEFAULT_FALSE_FLAG_NAMES = { 'check_i_know_what_i_am_doing', 'postgresql_databases[0].no_owner', 'healthchecks.create_slug', + 'repositories[0].append_only', + 'repositories[0].make_parent_dirs', } @@ -401,6 +403,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names arguments_group.add_argument( full_flag_name, action='store_true', + default=None, help=description, ) else: diff --git a/borgmatic/config/arguments.py b/borgmatic/config/arguments.py index 7a71c926..9aa83998 100644 --- a/borgmatic/config/arguments.py +++ b/borgmatic/config/arguments.py @@ -93,11 +93,16 @@ def convert_value_type(value, option_type): Given a string value and its schema type as a string, determine its logical type (string, boolean, integer, etc.), and return it converted to that type. - If the option type is a string, leave the value as a string so that special characters in it - don't get interpreted as YAML during conversion. + If the destination option type is a string, then leave the value as-is so that special + characters in it don't get interpreted as YAML during conversion. + + And if the source value isn't a string, return it as-is. Raise ruamel.yaml.error.YAMLError if there's a parse issue with the YAML. ''' + if not isinstance(value, str): + return value + if option_type == 'string': return value diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index fbc29c81..c40d9f9b 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -22,6 +22,9 @@ def insert_newline_before_comment(config, field_name): ) +SCALAR_SCHEMA_TYPES = {'string', 'boolean', 'integer', 'number'} + + def schema_to_sample_configuration(schema, source_config=None, level=0, parent_is_sequence=False): ''' Given a loaded configuration schema and a source configuration, generate and return sample @@ -33,9 +36,6 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i schema_type = schema.get('type') example = schema.get('example') - if example is not None: - return example - if schema_type == 'array' or (isinstance(schema_type, list) and 'array' in schema_type): config = ruamel.yaml.comments.CommentedSeq( [ @@ -53,7 +53,7 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i [ ( field_name, - schema_to_sample_configuration( + sub_schema.get('example') if field_name == 'source_directories' else schema_to_sample_configuration( sub_schema, (source_config or {}).get(field_name, {}), level + 1 ), ) @@ -64,6 +64,10 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i add_comments_to_configuration_object( config, schema, source_config, indent=indent, skip_first=parent_is_sequence ) + elif isinstance(schema_type, list) and all(element_schema_type in SCALAR_SCHEMA_TYPES for element_schema_type in schema_type): + return example + elif schema_type in SCALAR_SCHEMA_TYPES: + return example else: raise ValueError(f'Schema at level {level} is unsupported: {schema}') diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 909bc511..04051ece 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -53,6 +53,26 @@ properties: available encryption modes, run "borg init --help" with Borg 1 or "borg repo-create --help" with Borg 2. example: repokey-blake2 + append_only: + type: boolean + description: | + Whether the repository should be created append-only, + only used for the repo-create action. Defaults to false. + example: false + storage_quota: + type: string + description: | + The storage quota with which to create the repository, + only used for the repo-create action. Defaults to no + quota. + example: 5G + make_parent_dirs: + type: boolean + description: | + Whether any missing parent directories of the repository + path should be created, only used for the repo-create + action. Defaults to false. + example: true description: | A required list of local or remote repositories with paths and optional labels (which can be used with the --repository flag to From 1097a6576f48e7d06482dadc97182c2eed4ff4bd Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 19 Mar 2025 11:06:36 -0700 Subject: [PATCH 125/226] Add "progress" option to configuration (#303). --- borgmatic/actions/compact.py | 2 +- borgmatic/actions/create.py | 2 +- borgmatic/actions/extract.py | 2 +- borgmatic/borg/check.py | 5 +++-- borgmatic/borg/transfer.py | 2 +- borgmatic/commands/arguments.py | 1 + borgmatic/config/schema.yaml | 6 ++++++ 7 files changed, 14 insertions(+), 6 deletions(-) diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index a8ab6a6f..1a0cea3d 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -37,7 +37,7 @@ def run_compact( global_arguments, local_path=local_path, remote_path=remote_path, - progress=compact_arguments.progress, + progress=compact_arguments.progress or config.get('progress'), cleanup_commits=compact_arguments.cleanup_commits, threshold=compact_arguments.threshold, ) diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index ecc688b5..c12bf43e 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -327,7 +327,7 @@ def run_create( borgmatic_runtime_directory, local_path=local_path, remote_path=remote_path, - progress=create_arguments.progress, + progress=create_arguments.progress or config.get('progress'), stats=create_arguments.stats, json=create_arguments.json, list_files=create_arguments.list_files, diff --git a/borgmatic/actions/extract.py b/borgmatic/actions/extract.py index 6e2e7900..83f81176 100644 --- a/borgmatic/actions/extract.py +++ b/borgmatic/actions/extract.py @@ -45,5 +45,5 @@ def run_extract( remote_path=remote_path, destination_path=extract_arguments.destination, strip_components=extract_arguments.strip_components, - progress=extract_arguments.progress, + progress=extract_arguments.progress or config.get('progress'), ) diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 00a090a2..98c48946 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -143,6 +143,7 @@ def check_archives( umask = config.get('umask') borg_exit_codes = config.get('borg_exit_codes') working_directory = borgmatic.config.paths.get_working_directory(config) + progress = check_arguments.progress or config.get('progress') if 'data' in checks: checks.add('archives') @@ -170,7 +171,7 @@ def check_archives( + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + verbosity_flags - + (('--progress',) if check_arguments.progress else ()) + + (('--progress',) if progress else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + flags.make_repository_flags(repository_path, local_borg_version) ) @@ -180,7 +181,7 @@ def check_archives( # The Borg repair option triggers an interactive prompt, which won't work when output is # captured. And progress messes with the terminal directly. output_file=( - DO_NOT_CAPTURE if check_arguments.repair or check_arguments.progress else None + DO_NOT_CAPTURE if check_arguments.repair or progress else None ), environment=environment.make_environment(config), working_directory=working_directory, diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index 3af998a6..32dc48b3 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -56,7 +56,7 @@ def transfer_archives( return execute_command( full_command, output_log_level=logging.ANSWER, - output_file=DO_NOT_CAPTURE if transfer_arguments.progress else None, + output_file=DO_NOT_CAPTURE if (transfer_arguments.progress or config.get('progress')) else None, environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index bf65c24b..19b933d8 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -305,6 +305,7 @@ DEFAULT_FALSE_FLAG_NAMES = { 'healthchecks.create_slug', 'repositories[0].append_only', 'repositories[0].make_parent_dirs', + 'progress', } diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 04051ece..fbb2f5a4 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -793,6 +793,12 @@ properties: Apply color to console output. Can be overridden with --no-color command-line flag. Defaults to true. example: false + progress: + type: boolean + description: | + Display progress as each file or archive is processed. Defaults to + false. + example: true skip_actions: type: array items: From d02d31f445c2c06fc589abf7a8560cd15e0c050f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 19 Mar 2025 11:37:17 -0700 Subject: [PATCH 126/226] Use schema defaults instead of a flag name whitelist to make valueless boolean flags (#303). --- borgmatic/commands/arguments.py | 25 +++------------------ borgmatic/config/schema.yaml | 39 ++++++++++++++++++++++++++++----- 2 files changed, 36 insertions(+), 28 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 19b933d8..5d980a72 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -288,27 +288,6 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse ) -# As a UX nicety, allow boolean options that have a default of false to have command-line flags -# without values. -DEFAULT_FALSE_FLAG_NAMES = { - 'one_file_system', - 'numeric_ids', - 'read_special', - 'exclude_caches', - 'keep_exclude_tags', - 'exclude_nodump', - 'source_directories_must_exist', - 'relocated_repo_access_is_ok', - 'unknown_unencrypted_repo_access_is_ok', - 'check_i_know_what_i_am_doing', - 'postgresql_databases[0].no_owner', - 'healthchecks.create_slug', - 'repositories[0].append_only', - 'repositories[0].make_parent_dirs', - 'progress', -} - - def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names=None): ''' Given an argparse._ArgumentGroup instance, a configuration schema dict, and a sequence of @@ -400,7 +379,9 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names argument_type = borgmatic.config.schema.parse_type(schema_type) full_flag_name = f"--{flag_name.replace('_', '-')}" - if flag_name in DEFAULT_FALSE_FLAG_NAMES: + # As a UX nicety, allow boolean options that have a default of false to have command-line flags + # without values. + if schema_type == 'boolean' and schema.get('default') == False: arguments_group.add_argument( full_flag_name, action='store_true', diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index fbb2f5a4..481b9e78 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -58,7 +58,8 @@ properties: description: | Whether the repository should be created append-only, only used for the repo-create action. Defaults to false. - example: false + default: false + example: true storage_quota: type: string description: | @@ -72,6 +73,7 @@ properties: Whether any missing parent directories of the repository path should be created, only used for the repo-create action. Defaults to false. + default: false example: true description: | A required list of local or remote repositories with paths and @@ -101,12 +103,14 @@ properties: description: | Stay in same file system; do not cross mount points beyond the given source directories. Defaults to false. + default: false example: true numeric_ids: type: boolean description: | Only store/extract numeric user and group identifiers. Defaults to false. + default: false example: true atime: type: boolean @@ -117,11 +121,13 @@ properties: ctime: type: boolean description: Store ctime into archive. Defaults to true. + default: true example: false birthtime: type: boolean description: | Store birthtime (creation date) into archive. Defaults to true. + default: true example: false read_special: type: boolean @@ -131,13 +137,15 @@ properties: used when backing up special devices such as /dev/zero. Defaults to false. But when a database hook is used, the setting here is ignored and read_special is considered true. - example: false + default: false + example: true flags: type: boolean description: | Record filesystem flags (e.g. NODUMP, IMMUTABLE) in archive. Defaults to true. - example: true + default: true + example: false files_cache: type: string description: | @@ -210,6 +218,7 @@ properties: Exclude directories that contain a CACHEDIR.TAG file. See http://www.brynosaurus.com/cachedir/spec.html for details. Defaults to false. + default: false example: true exclude_if_present: type: array @@ -226,11 +235,13 @@ properties: If true, the exclude_if_present filename is included in backups. Defaults to false, meaning that the exclude_if_present filename is omitted from backups. + default: false example: true exclude_nodump: type: boolean description: | Exclude files with the NODUMP flag. Defaults to false. + default: false example: true borgmatic_source_directory: type: string @@ -262,6 +273,7 @@ properties: description: | If true, then source directories (and root pattern paths) must exist. If they don't, an error is raised. Defaults to false. + default: false example: true encryption_passcommand: type: string @@ -458,19 +470,22 @@ properties: type: boolean description: | Bypass Borg error about a repository that has been moved. Defaults - to not bypassing. + to false. + default: false example: true unknown_unencrypted_repo_access_is_ok: type: boolean description: | Bypass Borg error about a previously unknown unencrypted repository. - Defaults to not bypassing. + Defaults to false. + default: false example: true check_i_know_what_i_am_doing: type: boolean description: | Bypass Borg confirmation about check with repair option. Defaults to - an interactive prompt from Borg. + false and an interactive prompt from Borg. + default: false example: true extra_borg_options: type: object @@ -792,12 +807,14 @@ properties: description: | Apply color to console output. Can be overridden with --no-color command-line flag. Defaults to true. + default: true example: false progress: type: boolean description: | Display progress as each file or archive is processed. Defaults to false. + default: false example: true skip_actions: type: array @@ -1166,6 +1183,7 @@ properties: backup itself. Defaults to true. Changing this to false prevents "borgmatic bootstrap" from extracting configuration files from the backup. + default: true example: false description: | Support for the "borgmatic bootstrap" action, used to extract @@ -1250,6 +1268,7 @@ properties: schema elements. These statements will fail unless the initial connection to the database is made by a superuser. + default: false example: true format: type: string @@ -1488,6 +1507,7 @@ properties: Use the "--add-drop-database" flag with mariadb-dump, causing the database to be dropped right before restore. Defaults to true. + default: true example: false options: type: string @@ -1635,6 +1655,7 @@ properties: Use the "--add-drop-database" flag with mysqldump, causing the database to be dropped right before restore. Defaults to true. + default: true example: false options: type: string @@ -2335,6 +2356,7 @@ properties: description: | Send borgmatic logs to Apprise services as part the "finish", "fail", and "log" states. Defaults to true. + default: true example: false logs_size_limit: type: integer @@ -2440,12 +2462,14 @@ properties: description: | Verify the TLS certificate of the ping URL host. Defaults to true. + default: true example: false send_logs: type: boolean description: | Send borgmatic logs to Healthchecks as part the "finish", "fail", and "log" states. Defaults to true. + default: true example: false ping_body_limit: type: integer @@ -2478,6 +2502,7 @@ properties: the slug URL scheme (https://hc-ping.com// as opposed to https://hc-ping.com/). Defaults to false. + default: false example: true description: | Configuration for a monitoring integration with Healthchecks. Create @@ -2517,6 +2542,7 @@ properties: description: | Verify the TLS certificate of the push URL host. Defaults to true. + default: true example: false description: | Configuration for a monitoring integration with Uptime Kuma using @@ -2553,6 +2579,7 @@ properties: description: | Send borgmatic logs to PagerDuty when a backup errors. Defaults to true. + default: true example: false description: | Configuration for a monitoring integration with PagerDuty. Create an From 3e21cdb5796bfad45a9bb0c3e85ac1b51d6ca094 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 19 Mar 2025 19:43:04 -0700 Subject: [PATCH 127/226] Add "stats" option to configuration (#303). --- borgmatic/actions/create.py | 2 +- borgmatic/borg/prune.py | 5 +++-- borgmatic/config/schema.yaml | 7 +++++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index c12bf43e..47e915e5 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -328,7 +328,7 @@ def run_create( local_path=local_path, remote_path=remote_path, progress=create_arguments.progress or config.get('progress'), - stats=create_arguments.stats, + stats=create_arguments.stats or config.get('stats'), json=create_arguments.json, list_files=create_arguments.list_files, stream_processes=stream_processes, diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index 82a78201..38d0ed67 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -66,6 +66,7 @@ def prune_archives( borgmatic.logger.add_custom_log_levels() umask = config.get('umask', None) lock_wait = config.get('lock_wait', None) + stats = prune_arguments.stats or config.get('stats') extra_borg_options = config.get('extra_borg_options', {}).get('prune', '') full_command = ( @@ -77,7 +78,7 @@ def prune_archives( + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + ( ('--stats',) - if prune_arguments.stats + if stats and not dry_run and not feature.available(feature.Feature.NO_PRUNE_STATS, local_borg_version) else () @@ -94,7 +95,7 @@ def prune_archives( + flags.make_repository_flags(repository_path, local_borg_version) ) - if prune_arguments.stats or prune_arguments.list_archives: + if stats or prune_arguments.list_archives: output_log_level = logging.ANSWER else: output_log_level = logging.INFO diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 481b9e78..ab0ae1a3 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -816,6 +816,13 @@ properties: false. default: false example: true + stats: + type: boolean + description: | + Display statistics of an archive when running supported actions. + Defaults to false. + default: false + example: true skip_actions: type: array items: From ed6022d4a93b22921d1cb907e2d74e875bda1f18 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 19 Mar 2025 23:05:38 -0700 Subject: [PATCH 128/226] Add "list" option to configuration, corresponding to "--list" (#303). --- borgmatic/actions/create.py | 2 +- borgmatic/actions/export_tar.py | 2 +- borgmatic/borg/delete.py | 2 +- borgmatic/borg/prune.py | 2 +- borgmatic/config/schema.yaml | 32 ++++++++++++++++++++++---------- 5 files changed, 26 insertions(+), 14 deletions(-) diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index 47e915e5..c771f3d2 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -330,7 +330,7 @@ def run_create( progress=create_arguments.progress or config.get('progress'), stats=create_arguments.stats or config.get('stats'), json=create_arguments.json, - list_files=create_arguments.list_files, + list_files=create_arguments.list_files or config.get('list'), stream_processes=stream_processes, ) diff --git a/borgmatic/actions/export_tar.py b/borgmatic/actions/export_tar.py index d5c6bacb..1b41548b 100644 --- a/borgmatic/actions/export_tar.py +++ b/borgmatic/actions/export_tar.py @@ -43,6 +43,6 @@ def run_export_tar( local_path=local_path, remote_path=remote_path, tar_filter=export_tar_arguments.tar_filter, - list_files=export_tar_arguments.list_files, + list_files=export_tar_arguments.list_files or config.get('list'), strip_components=export_tar_arguments.strip_components, ) diff --git a/borgmatic/borg/delete.py b/borgmatic/borg/delete.py index d967582c..d5b02335 100644 --- a/borgmatic/borg/delete.py +++ b/borgmatic/borg/delete.py @@ -34,7 +34,7 @@ def make_delete_command( + borgmatic.borg.flags.make_flags('umask', config.get('umask')) + borgmatic.borg.flags.make_flags('log-json', global_arguments.log_json) + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait')) - + borgmatic.borg.flags.make_flags('list', delete_arguments.list_archives) + + borgmatic.borg.flags.make_flags('list', delete_arguments.list_archives or config.get('list')) + ( (('--force',) + (('--force',) if delete_arguments.force >= 2 else ())) if delete_arguments.force diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index 38d0ed67..5eef3a29 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -88,7 +88,7 @@ def prune_archives( prune_arguments, excludes=('repository', 'match_archives', 'stats', 'list_archives'), ) - + (('--list',) if prune_arguments.list_archives else ()) + + (('--list',) if prune_arguments.list_archives or config.get('list') else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (('--dry-run',) if dry_run else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index ab0ae1a3..71161990 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -49,30 +49,33 @@ properties: type: string description: | The encryption mode with which to create the repository, - only used for the repo-create action. To see the - available encryption modes, run "borg init --help" with - Borg 1 or "borg repo-create --help" with Borg 2. + only used for the repo-create action. Also set via the + "--encryption" flag. To see the available encryption + modes, run "borg init --help" with Borg 1 or "borg + repo-create --help" with Borg 2. example: repokey-blake2 append_only: type: boolean description: | Whether the repository should be created append-only, - only used for the repo-create action. Defaults to false. + only used for the repo-create action. Also set via the + "--append-only" flag. Defaults to false. default: false example: true storage_quota: type: string description: | The storage quota with which to create the repository, - only used for the repo-create action. Defaults to no - quota. + only used for the repo-create action. Also set via the + "--storage-quota" flag. Defaults to no quota. example: 5G make_parent_dirs: type: boolean description: | Whether any missing parent directories of the repository path should be created, only used for the repo-create - action. Defaults to false. + action. Also set via the "--make-parent-dirs" flag. + Defaults to false. default: false example: true description: | @@ -812,15 +815,24 @@ properties: progress: type: boolean description: | - Display progress as each file or archive is processed. Defaults to + Display progress as each file or archive is processed when running + supported actions. Also set via the "--progress" flag. Defaults to false. default: false example: true stats: type: boolean description: | - Display statistics of an archive when running supported actions. - Defaults to false. + Display statistics for an archive when running supported actions. + Also set via the "--stats" flag. Defaults to false. + default: false + example: true + list: + type: boolean + description: | + Display details for each file or archive as it is processed when + running supported actions. Also set via the "--list" flag. Defaults + to false. default: false example: true skip_actions: From 3119c924b4c97d36948e5dcaa0f2b48eb7c42fb6 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 19 Mar 2025 23:08:26 -0700 Subject: [PATCH 129/226] In configuration option descriptions, remove mention of corresponding CLI flags because it looks dumb on the command-line help (#303). --- borgmatic/config/schema.yaml | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 71161990..db4f4276 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -49,33 +49,30 @@ properties: type: string description: | The encryption mode with which to create the repository, - only used for the repo-create action. Also set via the - "--encryption" flag. To see the available encryption - modes, run "borg init --help" with Borg 1 or "borg - repo-create --help" with Borg 2. + only used for the repo-create action. To see the + available encryption modes, run "borg init --help" with + Borg 1 or "borg repo-create --help" with Borg 2. example: repokey-blake2 append_only: type: boolean description: | Whether the repository should be created append-only, - only used for the repo-create action. Also set via the - "--append-only" flag. Defaults to false. + only used for the repo-create action. Defaults to false. default: false example: true storage_quota: type: string description: | The storage quota with which to create the repository, - only used for the repo-create action. Also set via the - "--storage-quota" flag. Defaults to no quota. + only used for the repo-create action. Defaults to no + quota. example: 5G make_parent_dirs: type: boolean description: | Whether any missing parent directories of the repository path should be created, only used for the repo-create - action. Also set via the "--make-parent-dirs" flag. - Defaults to false. + action. Defaults to false. default: false example: true description: | @@ -816,23 +813,21 @@ properties: type: boolean description: | Display progress as each file or archive is processed when running - supported actions. Also set via the "--progress" flag. Defaults to - false. + supported actions. Defaults to false. default: false example: true stats: type: boolean description: | Display statistics for an archive when running supported actions. - Also set via the "--stats" flag. Defaults to false. + Defaults to false. default: false example: true list: type: boolean description: | Display details for each file or archive as it is processed when - running supported actions. Also set via the "--list" flag. Defaults - to false. + running supported actions. Defaults to false. default: false example: true skip_actions: From 624a7de622bb3a926997b37b7e9c40adc7922774 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Mar 2025 10:57:39 -0700 Subject: [PATCH 130/226] Document "after" command hooks running in case of error and make sure that happens in case of "before" hook error (#790). --- NEWS | 3 +++ borgmatic/hooks/command.py | 4 ++++ docs/how-to/add-preparation-and-cleanup-steps-to-backups.md | 5 +++++ tests/unit/hooks/test_command.py | 6 +++--- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index ef18be2b..df99734e 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,9 @@ * #790, #821: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more flexible "commands:". See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ + * #790: BREAKING: Run a configured "after" command hook even if an error occurs first. This + allows you to perform cleanup steps that correspond to "before" preparation commands—even when + something goes wrong. * #836: Add a custom command option for the SQLite hook. * #1010: When using Borg 2, don't pass the "--stats" flag to "borg prune". * #1020: Document a database use case involving a temporary database client container: diff --git a/borgmatic/hooks/command.py b/borgmatic/hooks/command.py index 14c766ca..9ea86f11 100644 --- a/borgmatic/hooks/command.py +++ b/borgmatic/hooks/command.py @@ -193,6 +193,10 @@ class Before_after_hooks: if considered_soft_failure(error): return + # Trigger the after hook manually, since raising here will prevent it from being run + # otherwise. + self.__exit__(None, None, None) + raise ValueError(f'Error running before {self.before_after} hook: {error}') def __exit__(self, exception_type, exception, traceback): diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index 8001a4c3..ec7ad7f3 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -66,6 +66,11 @@ Each command in the `commands:` list has the following options: * `when`: Only trigger the hook when borgmatic is run with particular actions (`create`, `prune`, etc.) listed here. Defaults to running for all actions. * `run`: List of one or more shell commands or scripts to run when this command hook is triggered. +An `after` command hook runs even if an error occurs in the corresponding `before` hook or between +those two hooks. This allows you to perform cleanup steps that correspond to `before` preparation +commands—even when something goes wrong. This is a departure from the way that the deprecated +`after_*` hooks worked. + There's also another command hook that works a little differently: ```yaml diff --git a/tests/unit/hooks/test_command.py b/tests/unit/hooks/test_command.py index d111418b..4662da7d 100644 --- a/tests/unit/hooks/test_command.py +++ b/tests/unit/hooks/test_command.py @@ -439,7 +439,7 @@ def test_before_after_hooks_calls_command_hooks(): pass -def test_before_after_hooks_with_before_error_raises_and_skips_after_hook(): +def test_before_after_hooks_with_before_error_runs_after_hook_and_raises(): commands = [ {'before': 'repository', 'run': ['foo', 'bar']}, {'after': 'repository', 'run': ['baz']}, @@ -455,8 +455,8 @@ def test_before_after_hooks_with_before_error_raises_and_skips_after_hook(): after='action', hook_name='myhook', action_names=['create'], - ).never() - flexmock(module).should_receive('execute_hooks').and_raise(OSError) + ).and_return(flexmock()).once() + flexmock(module).should_receive('execute_hooks').and_raise(OSError).and_return(None) flexmock(module).should_receive('considered_soft_failure').and_return(False) with pytest.raises(ValueError): From c2409d99684bb632be73a3a3cce5bc06dfe8630a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Mar 2025 11:13:37 -0700 Subject: [PATCH 131/226] Remove the "dump_data_sources" command hook, as it doesn't really solve the use case and works differently than all the other command hooks (#790). --- borgmatic/config/schema.yaml | 34 +-- borgmatic/hooks/command.py | 2 - borgmatic/hooks/data_source/bootstrap.py | 68 +++--- borgmatic/hooks/data_source/btrfs.py | 58 +++--- borgmatic/hooks/data_source/lvm.py | 132 ++++++------ borgmatic/hooks/data_source/mariadb.py | 100 +++++---- borgmatic/hooks/data_source/mongodb.py | 78 ++++--- borgmatic/hooks/data_source/mysql.py | 100 +++++---- borgmatic/hooks/data_source/postgresql.py | 194 ++++++++---------- borgmatic/hooks/data_source/sqlite.py | 93 ++++----- borgmatic/hooks/data_source/zfs.py | 110 +++++----- ...reparation-and-cleanup-steps-to-backups.md | 20 -- .../unit/hooks/data_source/test_bootstrap.py | 7 - tests/unit/hooks/data_source/test_btrfs.py | 18 -- tests/unit/hooks/data_source/test_lvm.py | 21 -- tests/unit/hooks/data_source/test_mariadb.py | 18 -- tests/unit/hooks/data_source/test_mongodb.py | 21 -- tests/unit/hooks/data_source/test_mysql.py | 18 -- .../unit/hooks/data_source/test_postgresql.py | 42 ---- tests/unit/hooks/data_source/test_sqlite.py | 21 -- tests/unit/hooks/data_source/test_zfs.py | 15 -- tests/unit/hooks/test_command.py | 115 ----------- 22 files changed, 428 insertions(+), 857 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 61c0c5e9..7b309ea0 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -959,7 +959,6 @@ properties: - repository - configuration - everything - - dump_data_sources description: | Name for the point in borgmatic's execution that the commands should be run before (required if @@ -972,19 +971,7 @@ properties: repositories in the current configuration file. * "everything" runs before all configuration files. - * "dump_data_sources" runs before each data - source is dumped. example: action - hooks: - type: array - items: - type: string - description: | - List of names of other hooks that this command - hook applies to. Defaults to all hooks of the - relevant type. Only supported for the - "dump_data_sources" hook. - example: postgresql when: type: array items: @@ -1013,9 +1000,7 @@ properties: - borg description: | List of actions for which the commands will be - run. Defaults to running for all actions. Ignored - for "dump_data_sources", which by its nature only - runs for "create". + run. Defaults to running for all actions. example: [create, prune, compact, check] run: type: array @@ -1037,7 +1022,6 @@ properties: - configuration - everything - error - - dump_data_sources description: | Name for the point in borgmatic's execution that the commands should be run after (required if @@ -1051,19 +1035,7 @@ properties: * "everything" runs after all configuration files. * "error" runs after an error occurs. - * "dump_data_sources" runs after each data - source is dumped. example: action - hooks: - type: array - items: - type: string - description: | - List of names of other hooks that this command - hook applies to. Defaults to all hooks of the - relevant type. Only supported for the - "dump_data_sources" hook. - example: postgresql when: type: array items: @@ -1093,9 +1065,7 @@ properties: description: | Only trigger the hook when borgmatic is run with particular actions listed here. Defaults to - running for all actions. Ignored for - "dump_data_sources", which by its nature only runs - for "create". + running for all actions. example: [create, prune, compact, check] run: type: array diff --git a/borgmatic/hooks/command.py b/borgmatic/hooks/command.py index 9ea86f11..c6fcc397 100644 --- a/borgmatic/hooks/command.py +++ b/borgmatic/hooks/command.py @@ -55,11 +55,9 @@ def filter_hooks(command_hooks, before=None, after=None, hook_name=None, action_ return tuple( hook_config for hook_config in command_hooks or () - for config_hook_names in (hook_config.get('hooks'),) for config_action_names in (hook_config.get('when'),) if before is None or hook_config.get('before') == before if after is None or hook_config.get('after') == after - if hook_name is None or config_hook_names is None or hook_name in config_hook_names if action_names is None or config_action_names is None or set(config_action_names or ()).intersection(set(action_names)) diff --git a/borgmatic/hooks/data_source/bootstrap.py b/borgmatic/hooks/data_source/bootstrap.py index e992cc2d..fe779f0e 100644 --- a/borgmatic/hooks/data_source/bootstrap.py +++ b/borgmatic/hooks/data_source/bootstrap.py @@ -6,7 +6,6 @@ import os import borgmatic.borg.pattern import borgmatic.config.paths -import borgmatic.hooks.command logger = logging.getLogger(__name__) @@ -38,46 +37,39 @@ def dump_data_sources( if hook_config and hook_config.get('store_config_files') is False: return [] - with borgmatic.hooks.command.Before_after_hooks( - command_hooks=config.get('commands'), - before_after='dump_data_sources', - umask=config.get('umask'), - dry_run=dry_run, - hook_name='bootstrap', - ): - borgmatic_manifest_path = os.path.join( - borgmatic_runtime_directory, 'bootstrap', 'manifest.json' - ) - - if dry_run: - return [] - - os.makedirs(os.path.dirname(borgmatic_manifest_path), exist_ok=True) - - with open(borgmatic_manifest_path, 'w') as manifest_file: - json.dump( - { - 'borgmatic_version': importlib.metadata.version('borgmatic'), - 'config_paths': config_paths, - }, - manifest_file, - ) - - patterns.extend( - borgmatic.borg.pattern.Pattern( - config_path, source=borgmatic.borg.pattern.Pattern_source.HOOK - ) - for config_path in config_paths - ) - patterns.append( - borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'bootstrap'), - source=borgmatic.borg.pattern.Pattern_source.HOOK, - ) - ) + borgmatic_manifest_path = os.path.join( + borgmatic_runtime_directory, 'bootstrap', 'manifest.json' + ) + if dry_run: return [] + os.makedirs(os.path.dirname(borgmatic_manifest_path), exist_ok=True) + + with open(borgmatic_manifest_path, 'w') as manifest_file: + json.dump( + { + 'borgmatic_version': importlib.metadata.version('borgmatic'), + 'config_paths': config_paths, + }, + manifest_file, + ) + + patterns.extend( + borgmatic.borg.pattern.Pattern( + config_path, source=borgmatic.borg.pattern.Pattern_source.HOOK + ) + for config_path in config_paths + ) + patterns.append( + borgmatic.borg.pattern.Pattern( + os.path.join(borgmatic_runtime_directory, 'bootstrap'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, + ) + ) + + return [] + def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, dry_run): ''' diff --git a/borgmatic/hooks/data_source/btrfs.py b/borgmatic/hooks/data_source/btrfs.py index 9cc8cb47..f46cbc2e 100644 --- a/borgmatic/hooks/data_source/btrfs.py +++ b/borgmatic/hooks/data_source/btrfs.py @@ -9,7 +9,6 @@ import subprocess import borgmatic.borg.pattern import borgmatic.config.paths import borgmatic.execute -import borgmatic.hooks.command import borgmatic.hooks.data_source.snapshot logger = logging.getLogger(__name__) @@ -250,48 +249,41 @@ def dump_data_sources( If this is a dry run, then don't actually snapshot anything. ''' - with borgmatic.hooks.command.Before_after_hooks( - command_hooks=config.get('commands'), - before_after='dump_data_sources', - umask=config.get('umask'), - dry_run=dry_run, - hook_name='btrfs', - ): - dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' - logger.info(f'Snapshotting Btrfs subvolumes{dry_run_label}') + dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' + logger.info(f'Snapshotting Btrfs subvolumes{dry_run_label}') - # Based on the configured patterns, determine Btrfs subvolumes to backup. Only consider those - # patterns that came from actual user configuration (as opposed to, say, other hooks). - btrfs_command = hook_config.get('btrfs_command', 'btrfs') - findmnt_command = hook_config.get('findmnt_command', 'findmnt') - subvolumes = get_subvolumes(btrfs_command, findmnt_command, patterns) + # Based on the configured patterns, determine Btrfs subvolumes to backup. Only consider those + # patterns that came from actual user configuration (as opposed to, say, other hooks). + btrfs_command = hook_config.get('btrfs_command', 'btrfs') + findmnt_command = hook_config.get('findmnt_command', 'findmnt') + subvolumes = get_subvolumes(btrfs_command, findmnt_command, patterns) - if not subvolumes: - logger.warning(f'No Btrfs subvolumes found to snapshot{dry_run_label}') + if not subvolumes: + logger.warning(f'No Btrfs subvolumes found to snapshot{dry_run_label}') - # Snapshot each subvolume, rewriting patterns to use their snapshot paths. - for subvolume in subvolumes: - logger.debug(f'Creating Btrfs snapshot for {subvolume.path} subvolume') + # Snapshot each subvolume, rewriting patterns to use their snapshot paths. + for subvolume in subvolumes: + logger.debug(f'Creating Btrfs snapshot for {subvolume.path} subvolume') - snapshot_path = make_snapshot_path(subvolume.path) + snapshot_path = make_snapshot_path(subvolume.path) - if dry_run: - continue + if dry_run: + continue - snapshot_subvolume(btrfs_command, subvolume.path, snapshot_path) + snapshot_subvolume(btrfs_command, subvolume.path, snapshot_path) - for pattern in subvolume.contained_patterns: - snapshot_pattern = make_borg_snapshot_pattern(subvolume.path, pattern) + for pattern in subvolume.contained_patterns: + snapshot_pattern = make_borg_snapshot_pattern(subvolume.path, pattern) - # Attempt to update the pattern in place, since pattern order matters to Borg. - try: - patterns[patterns.index(pattern)] = snapshot_pattern - except ValueError: - patterns.append(snapshot_pattern) + # Attempt to update the pattern in place, since pattern order matters to Borg. + try: + patterns[patterns.index(pattern)] = snapshot_pattern + except ValueError: + patterns.append(snapshot_pattern) - patterns.append(make_snapshot_exclude_pattern(subvolume.path)) + patterns.append(make_snapshot_exclude_pattern(subvolume.path)) - return [] + return [] def delete_snapshot(btrfs_command, snapshot_path): # pragma: no cover diff --git a/borgmatic/hooks/data_source/lvm.py b/borgmatic/hooks/data_source/lvm.py index 761a5aa6..11cd1c09 100644 --- a/borgmatic/hooks/data_source/lvm.py +++ b/borgmatic/hooks/data_source/lvm.py @@ -10,7 +10,6 @@ import subprocess import borgmatic.borg.pattern import borgmatic.config.paths import borgmatic.execute -import borgmatic.hooks.command import borgmatic.hooks.data_source.snapshot logger = logging.getLogger(__name__) @@ -198,84 +197,77 @@ def dump_data_sources( If this is a dry run, then don't actually snapshot anything. ''' - with borgmatic.hooks.command.Before_after_hooks( - command_hooks=config.get('commands'), - before_after='dump_data_sources', - umask=config.get('umask'), - dry_run=dry_run, - hook_name='lvm', - ): - dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' - logger.info(f'Snapshotting LVM logical volumes{dry_run_label}') + dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' + logger.info(f'Snapshotting LVM logical volumes{dry_run_label}') - # List logical volumes to get their mount points, but only consider those patterns that came - # from actual user configuration (as opposed to, say, other hooks). - lsblk_command = hook_config.get('lsblk_command', 'lsblk') - requested_logical_volumes = get_logical_volumes(lsblk_command, patterns) + # List logical volumes to get their mount points, but only consider those patterns that came + # from actual user configuration (as opposed to, say, other hooks). + lsblk_command = hook_config.get('lsblk_command', 'lsblk') + requested_logical_volumes = get_logical_volumes(lsblk_command, patterns) - # Snapshot each logical volume, rewriting source directories to use the snapshot paths. - snapshot_suffix = f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}' - normalized_runtime_directory = os.path.normpath(borgmatic_runtime_directory) + # Snapshot each logical volume, rewriting source directories to use the snapshot paths. + snapshot_suffix = f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}' + normalized_runtime_directory = os.path.normpath(borgmatic_runtime_directory) - if not requested_logical_volumes: - logger.warning(f'No LVM logical volumes found to snapshot{dry_run_label}') + if not requested_logical_volumes: + logger.warning(f'No LVM logical volumes found to snapshot{dry_run_label}') - for logical_volume in requested_logical_volumes: - snapshot_name = f'{logical_volume.name}_{snapshot_suffix}' - logger.debug( - f'Creating LVM snapshot {snapshot_name} of {logical_volume.mount_point}{dry_run_label}' + for logical_volume in requested_logical_volumes: + snapshot_name = f'{logical_volume.name}_{snapshot_suffix}' + logger.debug( + f'Creating LVM snapshot {snapshot_name} of {logical_volume.mount_point}{dry_run_label}' + ) + + if not dry_run: + snapshot_logical_volume( + hook_config.get('lvcreate_command', 'lvcreate'), + snapshot_name, + logical_volume.device_path, + hook_config.get('snapshot_size', DEFAULT_SNAPSHOT_SIZE), ) - if not dry_run: - snapshot_logical_volume( - hook_config.get('lvcreate_command', 'lvcreate'), - snapshot_name, - logical_volume.device_path, - hook_config.get('snapshot_size', DEFAULT_SNAPSHOT_SIZE), - ) + # Get the device path for the snapshot we just created. + try: + snapshot = get_snapshots( + hook_config.get('lvs_command', 'lvs'), snapshot_name=snapshot_name + )[0] + except IndexError: + raise ValueError(f'Cannot find LVM snapshot {snapshot_name}') - # Get the device path for the snapshot we just created. + # Mount the snapshot into a particular named temporary directory so that the snapshot ends + # up in the Borg archive at the "original" logical volume mount point path. + snapshot_mount_path = os.path.join( + normalized_runtime_directory, + 'lvm_snapshots', + hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest( + MOUNT_POINT_HASH_LENGTH + ), + logical_volume.mount_point.lstrip(os.path.sep), + ) + + logger.debug( + f'Mounting LVM snapshot {snapshot_name} at {snapshot_mount_path}{dry_run_label}' + ) + + if dry_run: + continue + + mount_snapshot( + hook_config.get('mount_command', 'mount'), snapshot.device_path, snapshot_mount_path + ) + + for pattern in logical_volume.contained_patterns: + snapshot_pattern = make_borg_snapshot_pattern( + pattern, logical_volume, normalized_runtime_directory + ) + + # Attempt to update the pattern in place, since pattern order matters to Borg. try: - snapshot = get_snapshots( - hook_config.get('lvs_command', 'lvs'), snapshot_name=snapshot_name - )[0] - except IndexError: - raise ValueError(f'Cannot find LVM snapshot {snapshot_name}') + patterns[patterns.index(pattern)] = snapshot_pattern + except ValueError: + patterns.append(snapshot_pattern) - # Mount the snapshot into a particular named temporary directory so that the snapshot ends - # up in the Borg archive at the "original" logical volume mount point path. - snapshot_mount_path = os.path.join( - normalized_runtime_directory, - 'lvm_snapshots', - hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest( - MOUNT_POINT_HASH_LENGTH - ), - logical_volume.mount_point.lstrip(os.path.sep), - ) - - logger.debug( - f'Mounting LVM snapshot {snapshot_name} at {snapshot_mount_path}{dry_run_label}' - ) - - if dry_run: - continue - - mount_snapshot( - hook_config.get('mount_command', 'mount'), snapshot.device_path, snapshot_mount_path - ) - - for pattern in logical_volume.contained_patterns: - snapshot_pattern = make_borg_snapshot_pattern( - pattern, logical_volume, normalized_runtime_directory - ) - - # Attempt to update the pattern in place, since pattern order matters to Borg. - try: - patterns[patterns.index(pattern)] = snapshot_pattern - except ValueError: - patterns.append(snapshot_pattern) - - return [] + return [] def unmount_snapshot(umount_command, snapshot_mount_path): # pragma: no cover diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index 3881d933..1f140793 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -6,7 +6,6 @@ import shlex import borgmatic.borg.pattern import borgmatic.config.paths -import borgmatic.hooks.command import borgmatic.hooks.credential.parse from borgmatic.execute import ( execute_command, @@ -243,78 +242,71 @@ def dump_data_sources( Also append the the parent directory of the database dumps to the given patterns list, so the dumps actually get backed up. ''' - with borgmatic.hooks.command.Before_after_hooks( - command_hooks=config.get('commands'), - before_after='dump_data_sources', - umask=config.get('umask'), - dry_run=dry_run, - hook_name='mariadb', - ): - dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' - processes = [] + dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' + processes = [] - logger.info(f'Dumping MariaDB databases{dry_run_label}') + logger.info(f'Dumping MariaDB databases{dry_run_label}') - for database in databases: - dump_path = make_dump_path(borgmatic_runtime_directory) - username = borgmatic.hooks.credential.parse.resolve_credential( - database.get('username'), config - ) - password = borgmatic.hooks.credential.parse.resolve_credential( - database.get('password'), config - ) - environment = dict(os.environ) - dump_database_names = database_names_to_dump( - database, config, username, password, environment, dry_run - ) + for database in databases: + dump_path = make_dump_path(borgmatic_runtime_directory) + username = borgmatic.hooks.credential.parse.resolve_credential( + database.get('username'), config + ) + password = borgmatic.hooks.credential.parse.resolve_credential( + database.get('password'), config + ) + environment = dict(os.environ) + dump_database_names = database_names_to_dump( + database, config, username, password, environment, dry_run + ) - if not dump_database_names: - if dry_run: - continue + if not dump_database_names: + if dry_run: + continue - raise ValueError('Cannot find any MariaDB databases to dump.') + raise ValueError('Cannot find any MariaDB databases to dump.') - if database['name'] == 'all' and database.get('format'): - for dump_name in dump_database_names: - renamed_database = copy.copy(database) - renamed_database['name'] = dump_name - processes.append( - execute_dump_command( - renamed_database, - config, - username, - password, - dump_path, - (dump_name,), - environment, - dry_run, - dry_run_label, - ) - ) - else: + if database['name'] == 'all' and database.get('format'): + for dump_name in dump_database_names: + renamed_database = copy.copy(database) + renamed_database['name'] = dump_name processes.append( execute_dump_command( - database, + renamed_database, config, username, password, dump_path, - dump_database_names, + (dump_name,), environment, dry_run, dry_run_label, ) ) - - if not dry_run: - patterns.append( - borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'mariadb_databases'), - source=borgmatic.borg.pattern.Pattern_source.HOOK, + else: + processes.append( + execute_dump_command( + database, + config, + username, + password, + dump_path, + dump_database_names, + environment, + dry_run, + dry_run_label, ) ) - return [process for process in processes if process] + if not dry_run: + patterns.append( + borgmatic.borg.pattern.Pattern( + os.path.join(borgmatic_runtime_directory, 'mariadb_databases'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, + ) + ) + + return [process for process in processes if process] def remove_data_source_dumps( diff --git a/borgmatic/hooks/data_source/mongodb.py b/borgmatic/hooks/data_source/mongodb.py index 033fa4ab..ff22d8c4 100644 --- a/borgmatic/hooks/data_source/mongodb.py +++ b/borgmatic/hooks/data_source/mongodb.py @@ -4,7 +4,6 @@ import shlex import borgmatic.borg.pattern import borgmatic.config.paths -import borgmatic.hooks.command import borgmatic.hooks.credential.parse from borgmatic.execute import execute_command, execute_command_with_processes from borgmatic.hooks.data_source import dump @@ -49,53 +48,46 @@ def dump_data_sources( Also append the the parent directory of the database dumps to the given patterns list, so the dumps actually get backed up. ''' - with borgmatic.hooks.command.Before_after_hooks( - command_hooks=config.get('commands'), - before_after='dump_data_sources', - umask=config.get('umask'), - dry_run=dry_run, - hook_name='mongodb', - ): - dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' + dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' - logger.info(f'Dumping MongoDB databases{dry_run_label}') + logger.info(f'Dumping MongoDB databases{dry_run_label}') - processes = [] + processes = [] - for database in databases: - name = database['name'] - dump_filename = dump.make_data_source_dump_filename( - make_dump_path(borgmatic_runtime_directory), - name, - database.get('hostname'), - database.get('port'), + for database in databases: + name = database['name'] + dump_filename = dump.make_data_source_dump_filename( + make_dump_path(borgmatic_runtime_directory), + name, + database.get('hostname'), + database.get('port'), + ) + dump_format = database.get('format', 'archive') + + logger.debug( + f'Dumping MongoDB database {name} to {dump_filename}{dry_run_label}', + ) + if dry_run: + continue + + command = build_dump_command(database, config, dump_filename, dump_format) + + if dump_format == 'directory': + dump.create_parent_directory_for_dump(dump_filename) + execute_command(command, shell=True) + else: + dump.create_named_pipe_for_dump(dump_filename) + processes.append(execute_command(command, shell=True, run_to_completion=False)) + + if not dry_run: + patterns.append( + borgmatic.borg.pattern.Pattern( + os.path.join(borgmatic_runtime_directory, 'mongodb_databases'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) - dump_format = database.get('format', 'archive') + ) - logger.debug( - f'Dumping MongoDB database {name} to {dump_filename}{dry_run_label}', - ) - if dry_run: - continue - - command = build_dump_command(database, config, dump_filename, dump_format) - - if dump_format == 'directory': - dump.create_parent_directory_for_dump(dump_filename) - execute_command(command, shell=True) - else: - dump.create_named_pipe_for_dump(dump_filename) - processes.append(execute_command(command, shell=True, run_to_completion=False)) - - if not dry_run: - patterns.append( - borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'mongodb_databases'), - source=borgmatic.borg.pattern.Pattern_source.HOOK, - ) - ) - - return processes + return processes def make_password_config_file(password): diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index 351a575f..7c5f84f2 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -5,7 +5,6 @@ import shlex import borgmatic.borg.pattern import borgmatic.config.paths -import borgmatic.hooks.command import borgmatic.hooks.credential.parse import borgmatic.hooks.data_source.mariadb from borgmatic.execute import ( @@ -170,78 +169,71 @@ def dump_data_sources( Also append the the parent directory of the database dumps to the given patterns list, so the dumps actually get backed up. ''' - with borgmatic.hooks.command.Before_after_hooks( - command_hooks=config.get('commands'), - before_after='dump_data_sources', - umask=config.get('umask'), - dry_run=dry_run, - hook_name='mysql', - ): - dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' - processes = [] + dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' + processes = [] - logger.info(f'Dumping MySQL databases{dry_run_label}') + logger.info(f'Dumping MySQL databases{dry_run_label}') - for database in databases: - dump_path = make_dump_path(borgmatic_runtime_directory) - username = borgmatic.hooks.credential.parse.resolve_credential( - database.get('username'), config - ) - password = borgmatic.hooks.credential.parse.resolve_credential( - database.get('password'), config - ) - environment = dict(os.environ) - dump_database_names = database_names_to_dump( - database, config, username, password, environment, dry_run - ) + for database in databases: + dump_path = make_dump_path(borgmatic_runtime_directory) + username = borgmatic.hooks.credential.parse.resolve_credential( + database.get('username'), config + ) + password = borgmatic.hooks.credential.parse.resolve_credential( + database.get('password'), config + ) + environment = dict(os.environ) + dump_database_names = database_names_to_dump( + database, config, username, password, environment, dry_run + ) - if not dump_database_names: - if dry_run: - continue + if not dump_database_names: + if dry_run: + continue - raise ValueError('Cannot find any MySQL databases to dump.') + raise ValueError('Cannot find any MySQL databases to dump.') - if database['name'] == 'all' and database.get('format'): - for dump_name in dump_database_names: - renamed_database = copy.copy(database) - renamed_database['name'] = dump_name - processes.append( - execute_dump_command( - renamed_database, - config, - username, - password, - dump_path, - (dump_name,), - environment, - dry_run, - dry_run_label, - ) - ) - else: + if database['name'] == 'all' and database.get('format'): + for dump_name in dump_database_names: + renamed_database = copy.copy(database) + renamed_database['name'] = dump_name processes.append( execute_dump_command( - database, + renamed_database, config, username, password, dump_path, - dump_database_names, + (dump_name,), environment, dry_run, dry_run_label, ) ) - - if not dry_run: - patterns.append( - borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'mysql_databases'), - source=borgmatic.borg.pattern.Pattern_source.HOOK, + else: + processes.append( + execute_dump_command( + database, + config, + username, + password, + dump_path, + dump_database_names, + environment, + dry_run, + dry_run_label, ) ) - return [process for process in processes if process] + if not dry_run: + patterns.append( + borgmatic.borg.pattern.Pattern( + os.path.join(borgmatic_runtime_directory, 'mysql_databases'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, + ) + ) + + return [process for process in processes if process] def remove_data_source_dumps( diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index b25704f7..4f470c52 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -7,7 +7,6 @@ import shlex import borgmatic.borg.pattern import borgmatic.config.paths -import borgmatic.hooks.command import borgmatic.hooks.credential.parse from borgmatic.execute import ( execute_command, @@ -142,127 +141,112 @@ def dump_data_sources( Raise ValueError if the databases to dump cannot be determined. ''' - with borgmatic.hooks.command.Before_after_hooks( - command_hooks=config.get('commands'), - before_after='dump_data_sources', - umask=config.get('umask'), - dry_run=dry_run, - hook_name='postgresql', - ): - dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' - processes = [] + dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' + processes = [] - logger.info(f'Dumping PostgreSQL databases{dry_run_label}') + logger.info(f'Dumping PostgreSQL databases{dry_run_label}') - for database in databases: - environment = make_environment(database, config) - dump_path = make_dump_path(borgmatic_runtime_directory) - dump_database_names = database_names_to_dump(database, config, environment, dry_run) + for database in databases: + environment = make_environment(database, config) + dump_path = make_dump_path(borgmatic_runtime_directory) + dump_database_names = database_names_to_dump(database, config, environment, dry_run) - if not dump_database_names: - if dry_run: - continue + if not dump_database_names: + if dry_run: + continue - raise ValueError('Cannot find any PostgreSQL databases to dump.') + raise ValueError('Cannot find any PostgreSQL databases to dump.') - for database_name in dump_database_names: - dump_format = database.get('format', None if database_name == 'all' else 'custom') - compression = database.get('compression') - default_dump_command = 'pg_dumpall' if database_name == 'all' else 'pg_dump' - dump_command = tuple( - shlex.quote(part) - for part in shlex.split(database.get('pg_dump_command') or default_dump_command) + for database_name in dump_database_names: + dump_format = database.get('format', None if database_name == 'all' else 'custom') + compression = database.get('compression') + default_dump_command = 'pg_dumpall' if database_name == 'all' else 'pg_dump' + dump_command = tuple( + shlex.quote(part) + for part in shlex.split(database.get('pg_dump_command') or default_dump_command) + ) + dump_filename = dump.make_data_source_dump_filename( + dump_path, + database_name, + database.get('hostname'), + database.get('port'), + ) + if os.path.exists(dump_filename): + logger.warning( + f'Skipping duplicate dump of PostgreSQL database "{database_name}" to {dump_filename}' ) - dump_filename = dump.make_data_source_dump_filename( - dump_path, - database_name, - database.get('hostname'), - database.get('port'), - ) - if os.path.exists(dump_filename): - logger.warning( - f'Skipping duplicate dump of PostgreSQL database "{database_name}" to {dump_filename}' - ) - continue + continue - command = ( - dump_command - + ( - '--no-password', - '--clean', - '--if-exists', - ) - + ( - ('--host', shlex.quote(database['hostname'])) - if 'hostname' in database - else () - ) - + (('--port', shlex.quote(str(database['port']))) if 'port' in database else ()) - + ( - ( - '--username', - shlex.quote( - borgmatic.hooks.credential.parse.resolve_credential( - database['username'], config - ) - ), - ) - if 'username' in database - else () - ) - + (('--no-owner',) if database.get('no_owner', False) else ()) - + (('--format', shlex.quote(dump_format)) if dump_format else ()) - + ( - ('--compress', shlex.quote(str(compression))) - if compression is not None - else () - ) - + (('--file', shlex.quote(dump_filename)) if dump_format == 'directory' else ()) - + ( - tuple(shlex.quote(option) for option in database['options'].split(' ')) - if 'options' in database - else () - ) - + (() if database_name == 'all' else (shlex.quote(database_name),)) - # Use shell redirection rather than the --file flag to sidestep synchronization issues - # when pg_dump/pg_dumpall tries to write to a named pipe. But for the directory dump - # format in a particular, a named destination is required, and redirection doesn't work. - + (('>', shlex.quote(dump_filename)) if dump_format != 'directory' else ()) + command = ( + dump_command + + ( + '--no-password', + '--clean', + '--if-exists', ) - - logger.debug( - f'Dumping PostgreSQL database "{database_name}" to {dump_filename}{dry_run_label}' + + (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ()) + + (('--port', shlex.quote(str(database['port']))) if 'port' in database else ()) + + ( + ( + '--username', + shlex.quote( + borgmatic.hooks.credential.parse.resolve_credential( + database['username'], config + ) + ), + ) + if 'username' in database + else () ) - if dry_run: - continue + + (('--no-owner',) if database.get('no_owner', False) else ()) + + (('--format', shlex.quote(dump_format)) if dump_format else ()) + + (('--compress', shlex.quote(str(compression))) if compression is not None else ()) + + (('--file', shlex.quote(dump_filename)) if dump_format == 'directory' else ()) + + ( + tuple(shlex.quote(option) for option in database['options'].split(' ')) + if 'options' in database + else () + ) + + (() if database_name == 'all' else (shlex.quote(database_name),)) + # Use shell redirection rather than the --file flag to sidestep synchronization issues + # when pg_dump/pg_dumpall tries to write to a named pipe. But for the directory dump + # format in a particular, a named destination is required, and redirection doesn't work. + + (('>', shlex.quote(dump_filename)) if dump_format != 'directory' else ()) + ) - if dump_format == 'directory': - dump.create_parent_directory_for_dump(dump_filename) + logger.debug( + f'Dumping PostgreSQL database "{database_name}" to {dump_filename}{dry_run_label}' + ) + if dry_run: + continue + + if dump_format == 'directory': + dump.create_parent_directory_for_dump(dump_filename) + execute_command( + command, + shell=True, + environment=environment, + ) + else: + dump.create_named_pipe_for_dump(dump_filename) + processes.append( execute_command( command, shell=True, environment=environment, + run_to_completion=False, ) - else: - dump.create_named_pipe_for_dump(dump_filename) - processes.append( - execute_command( - command, - shell=True, - environment=environment, - run_to_completion=False, - ) - ) - - if not dry_run: - patterns.append( - borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'postgresql_databases'), - source=borgmatic.borg.pattern.Pattern_source.HOOK, ) - ) - return processes + if not dry_run: + patterns.append( + borgmatic.borg.pattern.Pattern( + os.path.join(borgmatic_runtime_directory, 'postgresql_databases'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, + ) + ) + + return processes def remove_data_source_dumps( diff --git a/borgmatic/hooks/data_source/sqlite.py b/borgmatic/hooks/data_source/sqlite.py index c7ed267b..347ca5dc 100644 --- a/borgmatic/hooks/data_source/sqlite.py +++ b/borgmatic/hooks/data_source/sqlite.py @@ -4,7 +4,6 @@ import shlex import borgmatic.borg.pattern import borgmatic.config.paths -import borgmatic.hooks.command from borgmatic.execute import execute_command, execute_command_with_processes from borgmatic.hooks.data_source import dump @@ -48,66 +47,58 @@ def dump_data_sources( Also append the the parent directory of the database dumps to the given patterns list, so the dumps actually get backed up. ''' - with borgmatic.hooks.command.Before_after_hooks( - command_hooks=config.get('commands'), - before_after='dump_data_sources', - umask=config.get('umask'), - dry_run=dry_run, - hook_name='sqlite', - ): - dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' - processes = [] + dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' + processes = [] - logger.info(f'Dumping SQLite databases{dry_run_label}') + logger.info(f'Dumping SQLite databases{dry_run_label}') - for database in databases: - database_path = database['path'] + for database in databases: + database_path = database['path'] - if database['name'] == 'all': - logger.warning('The "all" database name has no meaning for SQLite databases') - if not os.path.exists(database_path): - logger.warning( - f'No SQLite database at {database_path}; an empty database will be created and dumped' - ) - - dump_path = make_dump_path(borgmatic_runtime_directory) - dump_filename = dump.make_data_source_dump_filename(dump_path, database['name']) - - if os.path.exists(dump_filename): - logger.warning( - f'Skipping duplicate dump of SQLite database at {database_path} to {dump_filename}' - ) - continue - - sqlite_command = tuple( - shlex.quote(part) - for part in shlex.split(database.get('sqlite_command') or 'sqlite3') - ) - command = sqlite_command + ( - shlex.quote(database_path), - '.dump', - '>', - shlex.quote(dump_filename), + if database['name'] == 'all': + logger.warning('The "all" database name has no meaning for SQLite databases') + if not os.path.exists(database_path): + logger.warning( + f'No SQLite database at {database_path}; an empty database will be created and dumped' ) - logger.debug( - f'Dumping SQLite database at {database_path} to {dump_filename}{dry_run_label}' + dump_path = make_dump_path(borgmatic_runtime_directory) + dump_filename = dump.make_data_source_dump_filename(dump_path, database['name']) + + if os.path.exists(dump_filename): + logger.warning( + f'Skipping duplicate dump of SQLite database at {database_path} to {dump_filename}' ) - if dry_run: - continue + continue - dump.create_named_pipe_for_dump(dump_filename) - processes.append(execute_command(command, shell=True, run_to_completion=False)) + sqlite_command = tuple( + shlex.quote(part) for part in shlex.split(database.get('sqlite_command') or 'sqlite3') + ) + command = sqlite_command + ( + shlex.quote(database_path), + '.dump', + '>', + shlex.quote(dump_filename), + ) - if not dry_run: - patterns.append( - borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'sqlite_databases'), - source=borgmatic.borg.pattern.Pattern_source.HOOK, - ) + logger.debug( + f'Dumping SQLite database at {database_path} to {dump_filename}{dry_run_label}' + ) + if dry_run: + continue + + dump.create_named_pipe_for_dump(dump_filename) + processes.append(execute_command(command, shell=True, run_to_completion=False)) + + if not dry_run: + patterns.append( + borgmatic.borg.pattern.Pattern( + os.path.join(borgmatic_runtime_directory, 'sqlite_databases'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) + ) - return processes + return processes def remove_data_source_dumps( diff --git a/borgmatic/hooks/data_source/zfs.py b/borgmatic/hooks/data_source/zfs.py index c7073140..9cb77587 100644 --- a/borgmatic/hooks/data_source/zfs.py +++ b/borgmatic/hooks/data_source/zfs.py @@ -9,7 +9,6 @@ import subprocess import borgmatic.borg.pattern import borgmatic.config.paths import borgmatic.execute -import borgmatic.hooks.command import borgmatic.hooks.data_source.snapshot logger = logging.getLogger(__name__) @@ -244,71 +243,64 @@ def dump_data_sources( If this is a dry run, then don't actually snapshot anything. ''' - with borgmatic.hooks.command.Before_after_hooks( - command_hooks=config.get('commands'), - before_after='dump_data_sources', - umask=config.get('umask'), - dry_run=dry_run, - hook_name='zfs', - ): - dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' - logger.info(f'Snapshotting ZFS datasets{dry_run_label}') + dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' + logger.info(f'Snapshotting ZFS datasets{dry_run_label}') - # List ZFS datasets to get their mount points, but only consider those patterns that came from - # actual user configuration (as opposed to, say, other hooks). - zfs_command = hook_config.get('zfs_command', 'zfs') - requested_datasets = get_datasets_to_backup(zfs_command, patterns) + # List ZFS datasets to get their mount points, but only consider those patterns that came from + # actual user configuration (as opposed to, say, other hooks). + zfs_command = hook_config.get('zfs_command', 'zfs') + requested_datasets = get_datasets_to_backup(zfs_command, patterns) - # Snapshot each dataset, rewriting patterns to use the snapshot paths. - snapshot_name = f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}' - normalized_runtime_directory = os.path.normpath(borgmatic_runtime_directory) + # Snapshot each dataset, rewriting patterns to use the snapshot paths. + snapshot_name = f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}' + normalized_runtime_directory = os.path.normpath(borgmatic_runtime_directory) - if not requested_datasets: - logger.warning(f'No ZFS datasets found to snapshot{dry_run_label}') + if not requested_datasets: + logger.warning(f'No ZFS datasets found to snapshot{dry_run_label}') - for dataset in requested_datasets: - full_snapshot_name = f'{dataset.name}@{snapshot_name}' - logger.debug( - f'Creating ZFS snapshot {full_snapshot_name} of {dataset.mount_point}{dry_run_label}' + for dataset in requested_datasets: + full_snapshot_name = f'{dataset.name}@{snapshot_name}' + logger.debug( + f'Creating ZFS snapshot {full_snapshot_name} of {dataset.mount_point}{dry_run_label}' + ) + + if not dry_run: + snapshot_dataset(zfs_command, full_snapshot_name) + + # Mount the snapshot into a particular named temporary directory so that the snapshot ends + # up in the Borg archive at the "original" dataset mount point path. + snapshot_mount_path = os.path.join( + normalized_runtime_directory, + 'zfs_snapshots', + hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest( + MOUNT_POINT_HASH_LENGTH + ), + dataset.mount_point.lstrip(os.path.sep), + ) + + logger.debug( + f'Mounting ZFS snapshot {full_snapshot_name} at {snapshot_mount_path}{dry_run_label}' + ) + + if dry_run: + continue + + mount_snapshot( + hook_config.get('mount_command', 'mount'), full_snapshot_name, snapshot_mount_path + ) + + for pattern in dataset.contained_patterns: + snapshot_pattern = make_borg_snapshot_pattern( + pattern, dataset, normalized_runtime_directory ) - if not dry_run: - snapshot_dataset(zfs_command, full_snapshot_name) + # Attempt to update the pattern in place, since pattern order matters to Borg. + try: + patterns[patterns.index(pattern)] = snapshot_pattern + except ValueError: + patterns.append(snapshot_pattern) - # Mount the snapshot into a particular named temporary directory so that the snapshot ends - # up in the Borg archive at the "original" dataset mount point path. - snapshot_mount_path = os.path.join( - normalized_runtime_directory, - 'zfs_snapshots', - hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest( - MOUNT_POINT_HASH_LENGTH - ), - dataset.mount_point.lstrip(os.path.sep), - ) - - logger.debug( - f'Mounting ZFS snapshot {full_snapshot_name} at {snapshot_mount_path}{dry_run_label}' - ) - - if dry_run: - continue - - mount_snapshot( - hook_config.get('mount_command', 'mount'), full_snapshot_name, snapshot_mount_path - ) - - for pattern in dataset.contained_patterns: - snapshot_pattern = make_borg_snapshot_pattern( - pattern, dataset, normalized_runtime_directory - ) - - # Attempt to update the pattern in place, since pattern order matters to Borg. - try: - patterns[patterns.index(pattern)] = snapshot_pattern - except ValueError: - patterns.append(snapshot_pattern) - - return [] + return [] def unmount_snapshot(umount_command, snapshot_mount_path): # pragma: no cover diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index ec7ad7f3..096097b8 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -71,23 +71,6 @@ those two hooks. This allows you to perform cleanup steps that correspond to `be commands—even when something goes wrong. This is a departure from the way that the deprecated `after_*` hooks worked. -There's also another command hook that works a little differently: - -```yaml -commands: - - before: dump_data_sources - hooks: [postgresql] - run: - - echo "Right before the PostgreSQL database dump!" -``` - -This command hook has the following options: - - * `before` or `after`: Name for the point in borgmatic's execution that the commands should be run before or after: - * `dump_data_sources` runs before or after data sources are dumped (databases dumped or filesystems snapshotted) for each hook named in `hooks`. - * `hooks`: Names of other hooks that this command hook applies to, e.g. `postgresql`, `mariadb`, `zfs`, `btrfs`, etc. Defaults to all hooks of the relevant type. - * `run`: One or more shell commands or scripts to run when this command hook is triggered. - ### Order of execution @@ -102,9 +85,6 @@ borgmatic for the `create` and `prune` actions. Here's the order of execution: * Run `before: configuration` hooks (from the first configuration file). * Run `before: repository` hooks (for the first repository). * Run `before: action` hooks for `create`. - * Run `before: dump_data_sources` hooks (e.g. for the PostgreSQL hook). - * Actually dump data sources (e.g. PostgreSQL databases). - * Run `after: dump_data_sources` hooks (e.g. for the PostgreSQL hook). * Actually run the `create` action (e.g. `borg create`). * Run `after: action` hooks for `create`. * Run `before: action` hooks for `prune`. diff --git a/tests/unit/hooks/data_source/test_bootstrap.py b/tests/unit/hooks/data_source/test_bootstrap.py index f5299a4e..1623dfb7 100644 --- a/tests/unit/hooks/data_source/test_bootstrap.py +++ b/tests/unit/hooks/data_source/test_bootstrap.py @@ -6,9 +6,6 @@ from borgmatic.hooks.data_source import bootstrap as module def test_dump_data_sources_creates_manifest_file(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) flexmock(module.os).should_receive('makedirs') flexmock(module.importlib.metadata).should_receive('version').and_return('1.0.0') @@ -35,7 +32,6 @@ def test_dump_data_sources_creates_manifest_file(): def test_dump_data_sources_with_store_config_files_false_does_not_create_manifest_file(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').never() flexmock(module.os).should_receive('makedirs').never() flexmock(module.json).should_receive('dump').never() hook_config = {'store_config_files': False} @@ -51,9 +47,6 @@ def test_dump_data_sources_with_store_config_files_false_does_not_create_manifes def test_dump_data_sources_with_dry_run_does_not_create_manifest_file(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) flexmock(module.os).should_receive('makedirs').never() flexmock(module.json).should_receive('dump').never() diff --git a/tests/unit/hooks/data_source/test_btrfs.py b/tests/unit/hooks/data_source/test_btrfs.py index 974657d0..fb44b91d 100644 --- a/tests/unit/hooks/data_source/test_btrfs.py +++ b/tests/unit/hooks/data_source/test_btrfs.py @@ -269,9 +269,6 @@ def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_ def test_dump_data_sources_snapshots_each_subvolume_and_updates_patterns(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) patterns = [Pattern('/foo'), Pattern('/mnt/subvol1')] config = {'btrfs': {}} flexmock(module).should_receive('get_subvolumes').and_return( @@ -350,9 +347,6 @@ def test_dump_data_sources_snapshots_each_subvolume_and_updates_patterns(): def test_dump_data_sources_uses_custom_btrfs_command_in_commands(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) patterns = [Pattern('/foo'), Pattern('/mnt/subvol1')] config = {'btrfs': {'btrfs_command': '/usr/local/bin/btrfs'}} flexmock(module).should_receive('get_subvolumes').and_return( @@ -406,9 +400,6 @@ def test_dump_data_sources_uses_custom_btrfs_command_in_commands(): def test_dump_data_sources_uses_custom_findmnt_command_in_commands(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) patterns = [Pattern('/foo'), Pattern('/mnt/subvol1')] config = {'btrfs': {'findmnt_command': '/usr/local/bin/findmnt'}} flexmock(module).should_receive('get_subvolumes').with_args( @@ -464,9 +455,6 @@ def test_dump_data_sources_uses_custom_findmnt_command_in_commands(): def test_dump_data_sources_with_dry_run_skips_snapshot_and_patterns_update(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) patterns = [Pattern('/foo'), Pattern('/mnt/subvol1')] config = {'btrfs': {}} flexmock(module).should_receive('get_subvolumes').and_return( @@ -495,9 +483,6 @@ def test_dump_data_sources_with_dry_run_skips_snapshot_and_patterns_update(): def test_dump_data_sources_without_matching_subvolumes_skips_snapshot_and_patterns_update(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) patterns = [Pattern('/foo'), Pattern('/mnt/subvol1')] config = {'btrfs': {}} flexmock(module).should_receive('get_subvolumes').and_return(()) @@ -522,9 +507,6 @@ def test_dump_data_sources_without_matching_subvolumes_skips_snapshot_and_patter def test_dump_data_sources_snapshots_adds_to_existing_exclude_patterns(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) patterns = [Pattern('/foo'), Pattern('/mnt/subvol1')] config = {'btrfs': {}, 'exclude_patterns': ['/bar']} flexmock(module).should_receive('get_subvolumes').and_return( diff --git a/tests/unit/hooks/data_source/test_lvm.py b/tests/unit/hooks/data_source/test_lvm.py index 77655250..34cf5250 100644 --- a/tests/unit/hooks/data_source/test_lvm.py +++ b/tests/unit/hooks/data_source/test_lvm.py @@ -282,9 +282,6 @@ def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_ def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) config = {'lvm': {}} patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')] logical_volumes = ( @@ -354,9 +351,6 @@ def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns(): def test_dump_data_sources_with_no_logical_volumes_skips_snapshots(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) config = {'lvm': {}} patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')] flexmock(module).should_receive('get_logical_volumes').and_return(()) @@ -379,9 +373,6 @@ def test_dump_data_sources_with_no_logical_volumes_skips_snapshots(): def test_dump_data_sources_uses_snapshot_size_for_snapshot(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) config = {'lvm': {'snapshot_size': '1000PB'}} patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')] logical_volumes = ( @@ -457,9 +448,6 @@ def test_dump_data_sources_uses_snapshot_size_for_snapshot(): def test_dump_data_sources_uses_custom_commands(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) config = { 'lvm': { 'lsblk_command': '/usr/local/bin/lsblk', @@ -546,9 +534,6 @@ def test_dump_data_sources_uses_custom_commands(): def test_dump_data_sources_with_dry_run_skips_snapshots_and_does_not_touch_patterns(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) config = {'lvm': {}} patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')] flexmock(module).should_receive('get_logical_volumes').and_return( @@ -600,9 +585,6 @@ def test_dump_data_sources_with_dry_run_skips_snapshots_and_does_not_touch_patte def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained_patterns(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) config = {'lvm': {}} patterns = [Pattern('/hmm')] logical_volumes = ( @@ -673,9 +655,6 @@ def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained def test_dump_data_sources_with_missing_snapshot_errors(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) config = {'lvm': {}} patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')] flexmock(module).should_receive('get_logical_volumes').and_return( diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py index 6f9aaa30..871c1969 100644 --- a/tests/unit/hooks/data_source/test_mariadb.py +++ b/tests/unit/hooks/data_source/test_mariadb.py @@ -237,9 +237,6 @@ def test_use_streaming_false_for_no_databases(): def test_dump_data_sources_dumps_each_database(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'foo'}, {'name': 'bar'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') @@ -281,9 +278,6 @@ def test_dump_data_sources_dumps_each_database(): def test_dump_data_sources_dumps_with_password(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) database = {'name': 'foo', 'username': 'root', 'password': 'trustsome1'} process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') @@ -318,9 +312,6 @@ def test_dump_data_sources_dumps_with_password(): def test_dump_data_sources_dumps_all_databases_at_once(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'all'}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') @@ -352,9 +343,6 @@ def test_dump_data_sources_dumps_all_databases_at_once(): def test_dump_data_sources_dumps_all_databases_separately_when_format_configured(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'all', 'format': 'sql'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') @@ -862,9 +850,6 @@ def test_execute_dump_command_with_dry_run_skips_mariadb_dump(): def test_dump_data_sources_errors_for_missing_all_databases(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'all'}] flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) @@ -888,9 +873,6 @@ def test_dump_data_sources_errors_for_missing_all_databases(): def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'all'}] flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) diff --git a/tests/unit/hooks/data_source/test_mongodb.py b/tests/unit/hooks/data_source/test_mongodb.py index cd74e29f..0cef73f8 100644 --- a/tests/unit/hooks/data_source/test_mongodb.py +++ b/tests/unit/hooks/data_source/test_mongodb.py @@ -24,9 +24,6 @@ def test_use_streaming_false_for_no_databases(): def test_dump_data_sources_runs_mongodump_for_each_database(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'foo'}, {'name': 'bar'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') @@ -56,9 +53,6 @@ def test_dump_data_sources_runs_mongodump_for_each_database(): def test_dump_data_sources_with_dry_run_skips_mongodump(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'foo'}, {'name': 'bar'}] flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( @@ -81,9 +75,6 @@ def test_dump_data_sources_with_dry_run_skips_mongodump(): def test_dump_data_sources_runs_mongodump_with_hostname_and_port(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') @@ -120,9 +111,6 @@ def test_dump_data_sources_runs_mongodump_with_hostname_and_port(): def test_dump_data_sources_runs_mongodump_with_username_and_password(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [ { 'name': 'foo', @@ -174,9 +162,6 @@ def test_dump_data_sources_runs_mongodump_with_username_and_password(): def test_dump_data_sources_runs_mongodump_with_directory_format(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'foo', 'format': 'directory'}] flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( @@ -204,9 +189,6 @@ def test_dump_data_sources_runs_mongodump_with_directory_format(): def test_dump_data_sources_runs_mongodump_with_options(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'foo', 'options': '--stuff=such'}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') @@ -240,9 +222,6 @@ def test_dump_data_sources_runs_mongodump_with_options(): def test_dump_data_sources_runs_mongodumpall_for_all_databases(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'all'}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') diff --git a/tests/unit/hooks/data_source/test_mysql.py b/tests/unit/hooks/data_source/test_mysql.py index c6d40155..63bb7a72 100644 --- a/tests/unit/hooks/data_source/test_mysql.py +++ b/tests/unit/hooks/data_source/test_mysql.py @@ -134,9 +134,6 @@ def test_use_streaming_false_for_no_databases(): def test_dump_data_sources_dumps_each_database(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'foo'}, {'name': 'bar'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') @@ -175,9 +172,6 @@ def test_dump_data_sources_dumps_each_database(): def test_dump_data_sources_dumps_with_password(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) database = {'name': 'foo', 'username': 'root', 'password': 'trustsome1'} process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') @@ -212,9 +206,6 @@ def test_dump_data_sources_dumps_with_password(): def test_dump_data_sources_dumps_all_databases_at_once(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'all'}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') @@ -246,9 +237,6 @@ def test_dump_data_sources_dumps_all_databases_at_once(): def test_dump_data_sources_dumps_all_databases_separately_when_format_configured(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'all', 'format': 'sql'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') @@ -774,9 +762,6 @@ def test_execute_dump_command_with_dry_run_skips_mysqldump(): def test_dump_data_sources_errors_for_missing_all_databases(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'all'}] flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) @@ -800,9 +785,6 @@ def test_dump_data_sources_errors_for_missing_all_databases(): def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'all'}] flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) diff --git a/tests/unit/hooks/data_source/test_postgresql.py b/tests/unit/hooks/data_source/test_postgresql.py index da73c4e6..f1589a6e 100644 --- a/tests/unit/hooks/data_source/test_postgresql.py +++ b/tests/unit/hooks/data_source/test_postgresql.py @@ -236,9 +236,6 @@ def test_use_streaming_false_for_no_databases(): def test_dump_data_sources_runs_pg_dump_for_each_database(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'foo'}, {'name': 'bar'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) @@ -287,9 +284,6 @@ def test_dump_data_sources_runs_pg_dump_for_each_database(): def test_dump_data_sources_raises_when_no_database_names_to_dump(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'foo'}, {'name': 'bar'}] flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') @@ -307,9 +301,6 @@ def test_dump_data_sources_raises_when_no_database_names_to_dump(): def test_dump_data_sources_does_not_raise_when_no_database_names_to_dump(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'foo'}, {'name': 'bar'}] flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') @@ -326,9 +317,6 @@ def test_dump_data_sources_does_not_raise_when_no_database_names_to_dump(): def test_dump_data_sources_with_duplicate_dump_skips_pg_dump(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'foo'}, {'name': 'bar'}] flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') @@ -356,9 +344,6 @@ def test_dump_data_sources_with_duplicate_dump_skips_pg_dump(): def test_dump_data_sources_with_dry_run_skips_pg_dump(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'foo'}, {'name': 'bar'}] flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') @@ -389,9 +374,6 @@ def test_dump_data_sources_with_dry_run_skips_pg_dump(): def test_dump_data_sources_runs_pg_dump_with_hostname_and_port(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] process = flexmock() flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) @@ -438,9 +420,6 @@ def test_dump_data_sources_runs_pg_dump_with_hostname_and_port(): def test_dump_data_sources_runs_pg_dump_with_username_and_password(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1'}] process = flexmock() flexmock(module).should_receive('make_environment').and_return( @@ -487,9 +466,6 @@ def test_dump_data_sources_runs_pg_dump_with_username_and_password(): def test_dump_data_sources_with_username_injection_attack_gets_escaped(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'foo', 'username': 'postgres; naughty-command', 'password': 'trustsome1'}] process = flexmock() flexmock(module).should_receive('make_environment').and_return( @@ -536,9 +512,6 @@ def test_dump_data_sources_with_username_injection_attack_gets_escaped(): def test_dump_data_sources_runs_pg_dump_with_directory_format(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'foo', 'format': 'directory'}] flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') @@ -583,9 +556,6 @@ def test_dump_data_sources_runs_pg_dump_with_directory_format(): def test_dump_data_sources_runs_pg_dump_with_string_compression(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'foo', 'compression': 'winrar'}] processes = [flexmock()] flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) @@ -633,9 +603,6 @@ def test_dump_data_sources_runs_pg_dump_with_string_compression(): def test_dump_data_sources_runs_pg_dump_with_integer_compression(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'foo', 'compression': 0}] processes = [flexmock()] flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) @@ -683,9 +650,6 @@ def test_dump_data_sources_runs_pg_dump_with_integer_compression(): def test_dump_data_sources_runs_pg_dump_with_options(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'foo', 'options': '--stuff=such'}] process = flexmock() flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) @@ -729,9 +693,6 @@ def test_dump_data_sources_runs_pg_dump_with_options(): def test_dump_data_sources_runs_pg_dumpall_for_all_databases(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'all'}] process = flexmock() flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) @@ -764,9 +725,6 @@ def test_dump_data_sources_runs_pg_dumpall_for_all_databases(): def test_dump_data_sources_runs_non_default_pg_dump(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'foo', 'pg_dump_command': 'special_pg_dump --compress *'}] process = flexmock() flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) diff --git a/tests/unit/hooks/data_source/test_sqlite.py b/tests/unit/hooks/data_source/test_sqlite.py index b54af4ef..16a4e8f9 100644 --- a/tests/unit/hooks/data_source/test_sqlite.py +++ b/tests/unit/hooks/data_source/test_sqlite.py @@ -17,9 +17,6 @@ def test_use_streaming_false_for_no_databases(): def test_dump_data_sources_logs_and_skips_if_dump_already_exists(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'path': '/path/to/database', 'name': 'database'}] flexmock(module).should_receive('make_dump_path').and_return('/run/borgmatic') @@ -44,9 +41,6 @@ def test_dump_data_sources_logs_and_skips_if_dump_already_exists(): def test_dump_data_sources_dumps_each_database(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [ {'path': '/path/to/database1', 'name': 'database1'}, {'path': '/path/to/database2', 'name': 'database2'}, @@ -77,9 +71,6 @@ def test_dump_data_sources_dumps_each_database(): def test_dump_data_sources_with_path_injection_attack_gets_escaped(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [ {'path': '/path/to/database1; naughty-command', 'name': 'database1'}, ] @@ -117,9 +108,6 @@ def test_dump_data_sources_with_path_injection_attack_gets_escaped(): def test_dump_data_sources_runs_non_default_sqlite_with_path_injection_attack_gets_escaped(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [ { 'path': '/path/to/database1; naughty-command', @@ -162,9 +150,6 @@ def test_dump_data_sources_runs_non_default_sqlite_with_path_injection_attack_ge def test_dump_data_sources_with_non_existent_path_warns_and_dumps_database(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [ {'path': '/path/to/database1', 'name': 'database1'}, ] @@ -193,9 +178,6 @@ def test_dump_data_sources_with_non_existent_path_warns_and_dumps_database(): def test_dump_data_sources_with_name_all_warns_and_dumps_all_databases(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [ {'path': '/path/to/database1', 'name': 'all'}, ] @@ -226,9 +208,6 @@ def test_dump_data_sources_with_name_all_warns_and_dumps_all_databases(): def test_dump_data_sources_does_not_dump_if_dry_run(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'path': '/path/to/database', 'name': 'database'}] flexmock(module).should_receive('make_dump_path').and_return('/run/borgmatic') diff --git a/tests/unit/hooks/data_source/test_zfs.py b/tests/unit/hooks/data_source/test_zfs.py index f1304543..74ca87b3 100644 --- a/tests/unit/hooks/data_source/test_zfs.py +++ b/tests/unit/hooks/data_source/test_zfs.py @@ -296,9 +296,6 @@ def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_ def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) dataset = flexmock( name='dataset', mount_point='/mnt/dataset', @@ -341,9 +338,6 @@ def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns(): def test_dump_data_sources_with_no_datasets_skips_snapshots(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) flexmock(module).should_receive('get_datasets_to_backup').and_return(()) flexmock(module.os).should_receive('getpid').and_return(1234) flexmock(module).should_receive('snapshot_dataset').never() @@ -366,9 +360,6 @@ def test_dump_data_sources_with_no_datasets_skips_snapshots(): def test_dump_data_sources_uses_custom_commands(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) dataset = flexmock( name='dataset', mount_point='/mnt/dataset', @@ -418,9 +409,6 @@ def test_dump_data_sources_uses_custom_commands(): def test_dump_data_sources_with_dry_run_skips_commands_and_does_not_touch_patterns(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) flexmock(module).should_receive('get_datasets_to_backup').and_return( (flexmock(name='dataset', mount_point='/mnt/dataset'),) ) @@ -445,9 +433,6 @@ def test_dump_data_sources_with_dry_run_skips_commands_and_does_not_touch_patter def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained_patterns(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) dataset = flexmock( name='dataset', mount_point='/mnt/dataset', diff --git a/tests/unit/hooks/test_command.py b/tests/unit/hooks/test_command.py index 4662da7d..f4925b36 100644 --- a/tests/unit/hooks/test_command.py +++ b/tests/unit/hooks/test_command.py @@ -133,121 +133,6 @@ def test_make_environment_with_pyinstaller_and_LD_LIBRARY_PATH_ORIG_copies_it_in }, ), ), - ( - ( - { - 'before': 'dump_data_sources', - 'hooks': ['postgresql'], - 'run': ['foo'], - }, - { - 'before': 'dump_data_sources', - 'hooks': ['lvm'], - 'run': ['bar'], - }, - { - 'after': 'dump_data_sources', - 'hooks': ['lvm'], - 'run': ['baz'], - }, - ), - { - 'before': 'dump_data_sources', - 'hook_name': 'lvm', - }, - ( - { - 'before': 'dump_data_sources', - 'hooks': ['lvm'], - 'run': ['bar'], - }, - ), - ), - ( - ( - { - 'before': 'dump_data_sources', - 'run': ['foo'], - }, - { - 'before': 'dump_data_sources', - 'run': ['bar'], - }, - { - 'after': 'dump_data_sources', - 'run': ['baz'], - }, - ), - { - 'before': 'dump_data_sources', - 'hook_name': 'lvm', - }, - ( - { - 'before': 'dump_data_sources', - 'run': ['foo'], - }, - { - 'before': 'dump_data_sources', - 'run': ['bar'], - }, - ), - ), - ( - ( - { - 'before': 'dump_data_sources', - 'hooks': ['postgresql', 'zfs', 'lvm'], - 'run': ['foo'], - }, - ), - { - 'before': 'dump_data_sources', - 'hook_name': 'lvm', - }, - ( - { - 'before': 'dump_data_sources', - 'hooks': ['postgresql', 'zfs', 'lvm'], - 'run': ['foo'], - }, - ), - ), - ( - ( - { - 'before': 'action', - 'when': ['create'], - 'run': ['foo'], - }, - { - 'before': 'action', - 'when': ['prune'], - 'run': ['bar'], - }, - { - 'before': 'action', - 'when': ['compact'], - 'run': ['baz'], - }, - ), - { - 'before': 'action', - 'action_names': ['create', 'compact', 'extract'], - }, - ( - { - 'before': 'action', - 'when': ['create'], - 'run': ['foo'], - }, - { - 'before': 'action', - 'when': ['compact'], - 'run': ['baz'], - }, - ), - ), ( ( { From 5525b467ef957889ceca6e2afbb520441c513ccd Mon Sep 17 00:00:00 2001 From: Nish_ <120EE0980@nitrkl.ac.in> Date: Fri, 21 Mar 2025 00:47:45 +0530 Subject: [PATCH 132/226] add key import command Signed-off-by: Nish_ <120EE0980@nitrkl.ac.in> --- borgmatic/actions/import_key.py | 33 +++ borgmatic/borg/import_key.py | 71 +++++++ borgmatic/commands/arguments.py | 25 +++ borgmatic/commands/borgmatic.py | 11 + tests/integration/config/test_schema.py | 8 +- tests/unit/actions/test_import_key.py | 20 ++ tests/unit/borg/test_import_key.py | 265 ++++++++++++++++++++++++ tests/unit/commands/test_borgmatic.py | 20 ++ 8 files changed, 452 insertions(+), 1 deletion(-) create mode 100644 borgmatic/actions/import_key.py create mode 100644 borgmatic/borg/import_key.py create mode 100644 tests/unit/actions/test_import_key.py create mode 100644 tests/unit/borg/test_import_key.py diff --git a/borgmatic/actions/import_key.py b/borgmatic/actions/import_key.py new file mode 100644 index 00000000..42813efa --- /dev/null +++ b/borgmatic/actions/import_key.py @@ -0,0 +1,33 @@ +import logging + +import borgmatic.borg.import_key +import borgmatic.config.validate + +logger = logging.getLogger(__name__) + + +def run_import_key( + repository, + config, + local_borg_version, + import_arguments, + global_arguments, + local_path, + remote_path, +): + ''' + Run the "key import" action for the given repository. + ''' + if import_arguments.repository is None or borgmatic.config.validate.repositories_match( + repository, import_arguments.repository + ): + logger.info('Importing repository key') + borgmatic.borg.import_key.import_key( + repository['path'], + config, + local_borg_version, + import_arguments, + global_arguments, + local_path=local_path, + remote_path=remote_path, + ) diff --git a/borgmatic/borg/import_key.py b/borgmatic/borg/import_key.py new file mode 100644 index 00000000..f8dd0604 --- /dev/null +++ b/borgmatic/borg/import_key.py @@ -0,0 +1,71 @@ +import logging +import os + +import borgmatic.config.paths +import borgmatic.logger +from borgmatic.borg import environment, flags +from borgmatic.execute import DO_NOT_CAPTURE, execute_command + +logger = logging.getLogger(__name__) + + +def import_key( + repository_path, + config, + local_borg_version, + import_arguments, + global_arguments, + local_path='borg', + remote_path=None, +): + ''' + Given a local or remote repository path, a configuration dict, the local Borg version, import + arguments, and optional local and remote Borg paths, import the repository key from the + path indicated in the import arguments. + + If the path is empty or "-", then read the key from stdin. + + Raise FileNotFoundError if the path is given and it does not exist. + ''' + borgmatic.logger.add_custom_log_levels() + umask = config.get('umask', None) + lock_wait = config.get('lock_wait', None) + working_directory = borgmatic.config.paths.get_working_directory(config) + + if import_arguments.path and import_arguments.path != '-': + if not os.path.exists(os.path.join(working_directory or '', import_arguments.path)): + raise FileNotFoundError(f'Path {import_arguments.path} does not exist. Aborting.') + + input_file = None + else: + input_file = DO_NOT_CAPTURE + + full_command = ( + (local_path, 'key', 'import') + + (('--remote-path', remote_path) if remote_path else ()) + + (('--umask', str(umask)) if umask else ()) + + (('--log-json',) if global_arguments.log_json else ()) + + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + + flags.make_flags('paper', import_arguments.paper) + + flags.make_repository_flags( + repository_path, + local_borg_version, + ) + + ((import_arguments.path,) if input_file is None else ()) + ) + + if global_arguments.dry_run: + logger.info('Skipping key import (dry run)') + return + + execute_command( + full_command, + input_file=input_file, + output_log_level=logging.ANSWER, + environment=environment.make_environment(config), + working_directory=working_directory, + borg_local_path=local_path, + borg_exit_codes=config.get('borg_exit_codes'), + ) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index e95ec5f4..b0114b2a 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1479,6 +1479,31 @@ def make_parsers(): '-h', '--help', action='help', help='Show this help message and exit' ) + key_import_parser = key_parsers.add_parser( + 'import', + help='Import a copy of the repository key from backup', + description='Import a copy of the repository key from backup', + add_help=False, + ) + key_import_group = key_import_parser.add_argument_group('key import arguments') + key_import_group.add_argument( + '--paper', + action='store_true', + help='Import interactively from a backup done with ``--paper``', + ) + key_import_group.add_argument( + '--repository', + help='Path of repository to import the key from, defaults to the configured repository if there is only one, quoted globs supported', + ) + key_import_group.add_argument( + '--path', + metavar='PATH', + help='Path to import the key from backup, defaults to stdin', + ) + key_import_group.add_argument( + '-h', '--help', action='help', help='Show this help message and exit' + ) + key_change_passphrase_parser = key_parsers.add_parser( 'change-passphrase', help='Change the passphrase protecting the repository key', diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 5691df71..d46c6f51 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -20,6 +20,7 @@ import borgmatic.actions.create import borgmatic.actions.delete import borgmatic.actions.export_key import borgmatic.actions.export_tar +import borgmatic.actions.import_key import borgmatic.actions.extract import borgmatic.actions.info import borgmatic.actions.list @@ -528,6 +529,16 @@ def run_actions( local_path, remote_path, ) + elif action_name == 'import' and action_name not in skip_actions: + borgmatic.actions.import_key.run_import_key( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) elif action_name == 'change-passphrase' and action_name not in skip_actions: borgmatic.actions.change_passphrase.run_change_passphrase( repository, diff --git a/tests/integration/config/test_schema.py b/tests/integration/config/test_schema.py index 30eb1c47..5cf114b1 100644 --- a/tests/integration/config/test_schema.py +++ b/tests/integration/config/test_schema.py @@ -14,7 +14,13 @@ def test_schema_line_length_stays_under_limit(): assert len(line.rstrip('\n')) <= MAXIMUM_LINE_LENGTH -ACTIONS_MODULE_NAMES_TO_OMIT = {'arguments', 'change_passphrase', 'export_key', 'json'} +ACTIONS_MODULE_NAMES_TO_OMIT = { + 'arguments', + 'change_passphrase', + 'export_key', + 'import_key', + 'json', +} ACTIONS_MODULE_NAMES_TO_ADD = {'key', 'umount'} diff --git a/tests/unit/actions/test_import_key.py b/tests/unit/actions/test_import_key.py new file mode 100644 index 00000000..565e3df3 --- /dev/null +++ b/tests/unit/actions/test_import_key.py @@ -0,0 +1,20 @@ +from flexmock import flexmock + +from borgmatic.actions import import_key as module + + +def test_run_import_key_does_not_raise(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.import_key).should_receive('import_key') + import_arguments = flexmock(repository=flexmock()) + + module.run_import_key( + repository={'path': 'repo'}, + config={}, + local_borg_version=None, + import_arguments=import_arguments, + global_arguments=flexmock(), + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/borg/test_import_key.py b/tests/unit/borg/test_import_key.py new file mode 100644 index 00000000..a9e8878c --- /dev/null +++ b/tests/unit/borg/test_import_key.py @@ -0,0 +1,265 @@ +import logging + +import pytest +from flexmock import flexmock + +import borgmatic.logger +from borgmatic.borg import import_key as module + +from ..test_verbosity import insert_logging_mock + + +def insert_execute_command_mock( + command, input_file=module.DO_NOT_CAPTURE, working_directory=None, borg_exit_codes=None +): + borgmatic.logger.add_custom_log_levels() + + flexmock(module.environment).should_receive('make_environment') + flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return( + working_directory, + ) + flexmock(module).should_receive('execute_command').with_args( + command, + input_file=input_file, + output_log_level=module.logging.ANSWER, + environment=None, + working_directory=working_directory, + borg_local_path=command[0], + borg_exit_codes=borg_exit_codes, + ).once() + + +def test_import_key_calls_borg_with_required_flags(): + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + insert_execute_command_mock(('borg', 'key', 'import', 'repo')) + + module.import_key( + repository_path='repo', + config={}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path=None), + global_arguments=flexmock(dry_run=False, log_json=False), + ) + + +def test_import_key_calls_borg_with_local_path(): + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + insert_execute_command_mock(('borg1', 'key', 'import', 'repo')) + + module.import_key( + repository_path='repo', + config={}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path=None), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg1', + ) + + +def test_import_key_calls_borg_using_exit_codes(): + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + borg_exit_codes = flexmock() + insert_execute_command_mock(('borg', 'key', 'import', 'repo'), borg_exit_codes=borg_exit_codes) + + module.import_key( + repository_path='repo', + config={'borg_exit_codes': borg_exit_codes}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path=None), + global_arguments=flexmock(dry_run=False, log_json=False), + ) + + +def test_import_key_calls_borg_with_remote_path_flags(): + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + insert_execute_command_mock(('borg', 'key', 'import', '--remote-path', 'borg1', 'repo')) + + module.import_key( + repository_path='repo', + config={}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path=None), + global_arguments=flexmock(dry_run=False, log_json=False), + remote_path='borg1', + ) + + +def test_import_key_calls_borg_with_umask_flags(): + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + insert_execute_command_mock(('borg', 'key', 'import', '--umask', '0770', 'repo')) + + module.import_key( + repository_path='repo', + config={'umask': '0770'}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path=None), + global_arguments=flexmock(dry_run=False, log_json=False), + ) + + +def test_import_key_calls_borg_with_log_json_flags(): + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + insert_execute_command_mock(('borg', 'key', 'import', '--log-json', 'repo')) + + module.import_key( + repository_path='repo', + config={}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path=None), + global_arguments=flexmock(dry_run=False, log_json=True), + ) + + +def test_import_key_calls_borg_with_lock_wait_flags(): + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + insert_execute_command_mock(('borg', 'key', 'import', '--lock-wait', '5', 'repo')) + + module.import_key( + repository_path='repo', + config={'lock_wait': '5'}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path=None), + global_arguments=flexmock(dry_run=False, log_json=False), + ) + + +def test_import_key_with_log_info_calls_borg_with_info_parameter(): + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + insert_execute_command_mock(('borg', 'key', 'import', '--info', 'repo')) + insert_logging_mock(logging.INFO) + + module.import_key( + repository_path='repo', + config={}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path=None), + global_arguments=flexmock(dry_run=False, log_json=False), + ) + + +def test_import_key_with_log_debug_calls_borg_with_debug_flags(): + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + insert_execute_command_mock(('borg', 'key', 'import', '--debug', '--show-rc', 'repo')) + insert_logging_mock(logging.DEBUG) + + module.import_key( + repository_path='repo', + config={}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path=None), + global_arguments=flexmock(dry_run=False, log_json=False), + ) + + +def test_import_key_calls_borg_with_paper_flags(): + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + insert_execute_command_mock(('borg', 'key', 'import', '--paper', 'repo')) + + module.import_key( + repository_path='repo', + config={}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=True, path=None), + global_arguments=flexmock(dry_run=False, log_json=False), + ) + + +def test_import_key_calls_borg_with_path_argument(): + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').with_args('source').and_return(True) + insert_execute_command_mock(('borg', 'key', 'import', 'repo', 'source'), input_file=None) + + module.import_key( + repository_path='repo', + config={}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path='source'), + global_arguments=flexmock(dry_run=False, log_json=False), + ) + + +def test_import_key_with_non_existent_path_raises(): + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module).should_receive('execute_command').never() + + with pytest.raises(FileNotFoundError): + module.import_key( + repository_path='repo', + config={}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path='source'), + global_arguments=flexmock(dry_run=False, log_json=False), + ) + + +def test_import_key_with_stdin_path_calls_borg_without_path_argument(): + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + insert_execute_command_mock(('borg', 'key', 'import', 'repo')) + + module.import_key( + repository_path='repo', + config={}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path='-'), + global_arguments=flexmock(dry_run=False, log_json=False), + ) + + +def test_import_key_with_dry_run_skips_borg_call(): + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + flexmock(module).should_receive('execute_command').never() + + module.import_key( + repository_path='repo', + config={}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path=None), + global_arguments=flexmock(dry_run=True, log_json=False), + ) + + +def test_import_key_calls_borg_with_working_directory(): + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + insert_execute_command_mock(('borg', 'key', 'import', 'repo'), working_directory='/working/dir') + + module.import_key( + repository_path='repo', + config={'working_directory': '/working/dir'}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path=None), + global_arguments=flexmock(dry_run=False, log_json=False), + ) + + +def test_import_key_calls_borg_with_path_argument_and_working_directory(): + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').with_args('/working/dir/source').and_return( + True + ).once() + insert_execute_command_mock( + ('borg', 'key', 'import', 'repo', 'source'), + input_file=None, + working_directory='/working/dir', + ) + + module.import_key( + repository_path='repo', + config={'working_directory': '/working/dir'}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path='source'), + global_arguments=flexmock(dry_run=False, log_json=False), + ) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index c31cf904..160f4454 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -1390,6 +1390,26 @@ def test_run_actions_runs_export_key(): ) +def test_run_actions_runs_import_key(): + flexmock(module).should_receive('add_custom_log_levels') + flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) + flexmock(borgmatic.actions.import_key).should_receive('run_import_key').once() + + tuple( + module.run_actions( + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'import': flexmock()}, + config_filename=flexmock(), + config={'repositories': []}, + config_paths=[], + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), + repository={'path': 'repo'}, + ) + ) + + def test_run_actions_runs_change_passphrase(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) From 7d989f727dbb9a218cb548ecce236133f74a814b Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 20 Mar 2025 12:23:00 -0700 Subject: [PATCH 133/226] Don't auto-add CLI flags for configuration options that already have per-action CLI flags (#303). --- borgmatic/commands/arguments.py | 8 ++++++++ borgmatic/config/schema.yaml | 9 ++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 5d980a72..dc72535f 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -288,6 +288,9 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse ) +OMITTED_FLAG_NAMES = {'progress', 'stats', 'list'} + + def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names=None): ''' Given an argparse._ArgumentGroup instance, a configuration schema dict, and a sequence of @@ -376,6 +379,11 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names description = description.replace('%', '%%') + # These options already have corresponding flags on individual actions (like "create + # --progress"), so don't bother adding them to the global flags. + if flag_name in OMITTED_FLAG_NAMES: + return + argument_type = borgmatic.config.schema.parse_type(schema_type) full_flag_name = f"--{flag_name.replace('_', '-')}" diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 006b9436..86cc5255 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -813,21 +813,24 @@ properties: type: boolean description: | Display progress as each file or archive is processed when running - supported actions. Defaults to false. + supported actions. Corresponds to the "--progress" flag on those + actions. Defaults to false. default: false example: true stats: type: boolean description: | Display statistics for an archive when running supported actions. - Defaults to false. + Corresponds to the "--stats" flag on those actions. Defaults to + false. default: false example: true list: type: boolean description: | Display details for each file or archive as it is processed when - running supported actions. Defaults to false. + running supported actions. Corresponds to the "--list" flag on those + actions. Defaults to false. default: false example: true skip_actions: From 8aaa5ba8a67d8adb7068fca22fd4879bf4896577 Mon Sep 17 00:00:00 2001 From: Nish_ <120EE0980@nitrkl.ac.in> Date: Fri, 21 Mar 2025 19:26:12 +0530 Subject: [PATCH 134/226] minor changes Signed-off-by: Nish_ <120EE0980@nitrkl.ac.in> --- borgmatic/borg/import_key.py | 6 +++--- borgmatic/commands/arguments.py | 2 +- tests/unit/borg/test_import_key.py | 22 ++++++++++++++++++---- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/borgmatic/borg/import_key.py b/borgmatic/borg/import_key.py index f8dd0604..7f011ebc 100644 --- a/borgmatic/borg/import_key.py +++ b/borgmatic/borg/import_key.py @@ -25,7 +25,7 @@ def import_key( If the path is empty or "-", then read the key from stdin. - Raise FileNotFoundError if the path is given and it does not exist. + Raise ValueError if the path is given and it does not exist. ''' borgmatic.logger.add_custom_log_levels() umask = config.get('umask', None) @@ -34,7 +34,7 @@ def import_key( if import_arguments.path and import_arguments.path != '-': if not os.path.exists(os.path.join(working_directory or '', import_arguments.path)): - raise FileNotFoundError(f'Path {import_arguments.path} does not exist. Aborting.') + raise ValueError(f'Path {import_arguments.path} does not exist. Aborting.') input_file = None else: @@ -63,7 +63,7 @@ def import_key( execute_command( full_command, input_file=input_file, - output_log_level=logging.ANSWER, + output_log_level=logging.INFO, environment=environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index b0114b2a..1bd68894 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1489,7 +1489,7 @@ def make_parsers(): key_import_group.add_argument( '--paper', action='store_true', - help='Import interactively from a backup done with ``--paper``', + help='Import interactively from a backup done with --paper', ) key_import_group.add_argument( '--repository', diff --git a/tests/unit/borg/test_import_key.py b/tests/unit/borg/test_import_key.py index a9e8878c..dfbcd900 100644 --- a/tests/unit/borg/test_import_key.py +++ b/tests/unit/borg/test_import_key.py @@ -3,7 +3,6 @@ import logging import pytest from flexmock import flexmock -import borgmatic.logger from borgmatic.borg import import_key as module from ..test_verbosity import insert_logging_mock @@ -12,7 +11,6 @@ from ..test_verbosity import insert_logging_mock def insert_execute_command_mock( command, input_file=module.DO_NOT_CAPTURE, working_directory=None, borg_exit_codes=None ): - borgmatic.logger.add_custom_log_levels() flexmock(module.environment).should_receive('make_environment') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return( @@ -21,7 +19,7 @@ def insert_execute_command_mock( flexmock(module).should_receive('execute_command').with_args( command, input_file=input_file, - output_log_level=module.logging.ANSWER, + output_log_level=module.logging.INFO, environment=None, working_directory=working_directory, borg_local_path=command[0], @@ -30,6 +28,7 @@ def insert_execute_command_mock( def test_import_key_calls_borg_with_required_flags(): + flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.os.path).should_receive('exists').never() insert_execute_command_mock(('borg', 'key', 'import', 'repo')) @@ -44,6 +43,7 @@ def test_import_key_calls_borg_with_required_flags(): def test_import_key_calls_borg_with_local_path(): + flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.os.path).should_receive('exists').never() insert_execute_command_mock(('borg1', 'key', 'import', 'repo')) @@ -59,6 +59,7 @@ def test_import_key_calls_borg_with_local_path(): def test_import_key_calls_borg_using_exit_codes(): + flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.os.path).should_receive('exists').never() borg_exit_codes = flexmock() @@ -74,6 +75,7 @@ def test_import_key_calls_borg_using_exit_codes(): def test_import_key_calls_borg_with_remote_path_flags(): + flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.os.path).should_receive('exists').never() insert_execute_command_mock(('borg', 'key', 'import', '--remote-path', 'borg1', 'repo')) @@ -89,6 +91,7 @@ def test_import_key_calls_borg_with_remote_path_flags(): def test_import_key_calls_borg_with_umask_flags(): + flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.os.path).should_receive('exists').never() insert_execute_command_mock(('borg', 'key', 'import', '--umask', '0770', 'repo')) @@ -103,6 +106,7 @@ def test_import_key_calls_borg_with_umask_flags(): def test_import_key_calls_borg_with_log_json_flags(): + flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.os.path).should_receive('exists').never() insert_execute_command_mock(('borg', 'key', 'import', '--log-json', 'repo')) @@ -117,6 +121,7 @@ def test_import_key_calls_borg_with_log_json_flags(): def test_import_key_calls_borg_with_lock_wait_flags(): + flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.os.path).should_receive('exists').never() insert_execute_command_mock(('borg', 'key', 'import', '--lock-wait', '5', 'repo')) @@ -131,6 +136,7 @@ def test_import_key_calls_borg_with_lock_wait_flags(): def test_import_key_with_log_info_calls_borg_with_info_parameter(): + flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.os.path).should_receive('exists').never() insert_execute_command_mock(('borg', 'key', 'import', '--info', 'repo')) @@ -146,6 +152,7 @@ def test_import_key_with_log_info_calls_borg_with_info_parameter(): def test_import_key_with_log_debug_calls_borg_with_debug_flags(): + flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.os.path).should_receive('exists').never() insert_execute_command_mock(('borg', 'key', 'import', '--debug', '--show-rc', 'repo')) @@ -161,6 +168,7 @@ def test_import_key_with_log_debug_calls_borg_with_debug_flags(): def test_import_key_calls_borg_with_paper_flags(): + flexmock(module.flags).should_receive('make_flags').and_return(('--paper',)) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.os.path).should_receive('exists').never() insert_execute_command_mock(('borg', 'key', 'import', '--paper', 'repo')) @@ -175,6 +183,7 @@ def test_import_key_calls_borg_with_paper_flags(): def test_import_key_calls_borg_with_path_argument(): + flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.os.path).should_receive('exists').with_args('source').and_return(True) insert_execute_command_mock(('borg', 'key', 'import', 'repo', 'source'), input_file=None) @@ -189,11 +198,12 @@ def test_import_key_calls_borg_with_path_argument(): def test_import_key_with_non_existent_path_raises(): + flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module).should_receive('execute_command').never() - with pytest.raises(FileNotFoundError): + with pytest.raises(ValueError): module.import_key( repository_path='repo', config={}, @@ -204,6 +214,7 @@ def test_import_key_with_non_existent_path_raises(): def test_import_key_with_stdin_path_calls_borg_without_path_argument(): + flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.os.path).should_receive('exists').never() insert_execute_command_mock(('borg', 'key', 'import', 'repo')) @@ -218,6 +229,7 @@ def test_import_key_with_stdin_path_calls_borg_without_path_argument(): def test_import_key_with_dry_run_skips_borg_call(): + flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.os.path).should_receive('exists').never() flexmock(module).should_receive('execute_command').never() @@ -232,6 +244,7 @@ def test_import_key_with_dry_run_skips_borg_call(): def test_import_key_calls_borg_with_working_directory(): + flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.os.path).should_receive('exists').never() insert_execute_command_mock(('borg', 'key', 'import', 'repo'), working_directory='/working/dir') @@ -246,6 +259,7 @@ def test_import_key_calls_borg_with_working_directory(): def test_import_key_calls_borg_with_path_argument_and_working_directory(): + flexmock(module.flags).should_receive('make_flags').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) flexmock(module.os.path).should_receive('exists').with_args('/working/dir/source').and_return( True From cbfc0bead132c779ec9039ec04f17ef3d235561c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 21 Mar 2025 09:56:42 -0700 Subject: [PATCH 135/226] Exclude --match-archives from global flags since it already exists on several actions (#303). --- borgmatic/commands/arguments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index dc72535f..cb4bbcef 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -288,7 +288,7 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse ) -OMITTED_FLAG_NAMES = {'progress', 'stats', 'list'} +OMITTED_FLAG_NAMES = {'match_archives', 'progress', 'stats', 'list'} def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names=None): From 587d31de7caacc366dc9b5ea807446de0db65587 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 21 Mar 2025 10:53:06 -0700 Subject: [PATCH 136/226] Run all command hooks respecting the "working_directory" option if configured (#790). --- NEWS | 8 +-- borgmatic/commands/borgmatic.py | 7 +++ borgmatic/hooks/command.py | 18 ++++--- ...reparation-and-cleanup-steps-to-backups.md | 27 ++++++++-- tests/unit/hooks/test_command.py | 49 +++++++++++++++++-- 5 files changed, 91 insertions(+), 18 deletions(-) diff --git a/NEWS b/NEWS index df99734e..f7d06b2c 100644 --- a/NEWS +++ b/NEWS @@ -2,9 +2,11 @@ * #790, #821: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more flexible "commands:". See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ - * #790: BREAKING: Run a configured "after" command hook even if an error occurs first. This - allows you to perform cleanup steps that correspond to "before" preparation commands—even when - something goes wrong. + * #790: BREAKING: For both new and deprecated command hooks, run a configured "after" hook even if + an error occurs first. This allows you to perform cleanup steps that correspond to "before" + preparation commands—even when something goes wrong. + * #790: BREAKING: Run all command hooks (both new and deprecated) respecting the + "working_directory" option if configured, meaning that hook commands are run in that directory. * #836: Add a custom command option for the SQLite hook. * #1010: When using Borg 2, don't pass the "--stats" flag to "borg prune". * #1020: Document a database use case involving a temporary database client container: diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 5691df71..a33913e9 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -33,6 +33,7 @@ import borgmatic.actions.restore import borgmatic.actions.transfer import borgmatic.commands.completion.bash import borgmatic.commands.completion.fish +import borgmatic.config.paths from borgmatic.borg import umount as borg_umount from borgmatic.borg import version as borg_version from borgmatic.commands.arguments import parse_arguments @@ -207,6 +208,7 @@ def run_configuration(config_filename, config, config_paths, arguments): command_hooks=config.get('commands'), before_after='configuration', umask=config.get('umask'), + working_directory=borgmatic.config.paths.get_working_directory(config), dry_run=global_arguments.dry_run, action_names=arguments.keys(), configuration_filename=config_filename, @@ -286,6 +288,7 @@ def run_configuration(config_filename, config, config_paths, arguments): config.get('commands'), after='error', action_names=arguments.keys() ), config.get('umask'), + borgmatic.config.paths.get_working_directory(config), global_arguments.dry_run, configuration_filename=config_filename, log_file=arguments['global'].log_file or '', @@ -342,6 +345,7 @@ def run_actions( command_hooks=config.get('commands'), before_after='repository', umask=config.get('umask'), + working_directory=borgmatic.config.paths.get_working_directory(config), dry_run=global_arguments.dry_run, action_names=arguments.keys(), **hook_context, @@ -354,6 +358,7 @@ def run_actions( command_hooks=config.get('commands'), before_after='action', umask=config.get('umask'), + working_directory=borgmatic.config.paths.get_working_directory(config), dry_run=global_arguments.dry_run, action_names=arguments.keys(), **hook_context, @@ -854,6 +859,7 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): config.get('commands'), before='everything', action_names=arguments.keys() ), config.get('umask'), + borgmatic.config.paths.get_working_directory(config), arguments['global'].dry_run, configuration_filename=config_filename, log_file=arguments['global'].log_file or '', @@ -908,6 +914,7 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): config.get('commands'), after='everything', action_names=arguments.keys() ), config.get('umask'), + borgmatic.config.paths.get_working_directory(config), arguments['global'].dry_run, configuration_filename=config_filename, log_file=arguments['global'].log_file or '', diff --git a/borgmatic/hooks/command.py b/borgmatic/hooks/command.py index c6fcc397..eff1a752 100644 --- a/borgmatic/hooks/command.py +++ b/borgmatic/hooks/command.py @@ -64,11 +64,11 @@ def filter_hooks(command_hooks, before=None, after=None, hook_name=None, action_ ) -def execute_hooks(command_hooks, umask, dry_run, **context): +def execute_hooks(command_hooks, umask, working_directory, dry_run, **context): ''' - Given a sequence of command hook dicts from configuration, a umask to execute with (or None), - and whether this is a dry run, run the commands for each hook. Or don't run them if this is a - dry run. + Given a sequence of command hook dicts from configuration, a umask to execute with (or None), a + working directory to execute with, and whether this is a dry run, run the commands for each + hook. Or don't run them if this is a dry run. The context contains optional values interpolated by name into the hook commands. @@ -121,6 +121,7 @@ def execute_hooks(command_hooks, umask, dry_run, **context): ), shell=True, environment=make_environment(os.environ), + working_directory=working_directory, ) finally: if original_umask: @@ -153,6 +154,7 @@ class Before_after_hooks: command_hooks, before_after, umask, + working_directory, dry_run, hook_name=None, action_names=None, @@ -160,12 +162,14 @@ class Before_after_hooks: ): ''' Given a sequence of command hook configuration dicts, the before/after name, a umask to run - commands with, a dry run flag, the name of the calling hook, a sequence of action names, and - any context for the executed commands, save those data points for use below. + commands with, a working directory to run commands with, a dry run flag, the name of the + calling hook, a sequence of action names, and any context for the executed commands, save + those data points for use below. ''' self.command_hooks = command_hooks self.before_after = before_after self.umask = umask + self.working_directory = working_directory self.dry_run = dry_run self.hook_name = hook_name self.action_names = action_names @@ -184,6 +188,7 @@ class Before_after_hooks: action_names=self.action_names, ), self.umask, + self.working_directory, self.dry_run, **self.context, ) @@ -210,6 +215,7 @@ class Before_after_hooks: action_names=self.action_names, ), self.umask, + self.working_directory, self.dry_run, **self.context, ) diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index 096097b8..977e87f1 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -66,10 +66,15 @@ Each command in the `commands:` list has the following options: * `when`: Only trigger the hook when borgmatic is run with particular actions (`create`, `prune`, etc.) listed here. Defaults to running for all actions. * `run`: List of one or more shell commands or scripts to run when this command hook is triggered. -An `after` command hook runs even if an error occurs in the corresponding `before` hook or between -those two hooks. This allows you to perform cleanup steps that correspond to `before` preparation -commands—even when something goes wrong. This is a departure from the way that the deprecated -`after_*` hooks worked. +An `after` command hook runs even if an error occurs in the corresponding +`before` hook or between those two hooks. This allows you to perform cleanup +steps that correspond to `before` preparation commands—even when something goes +wrong. This is a departure from the way that the deprecated `after_*` hooks +worked in borgmatic prior to version 2.0.0. + +Additionally, when command hooks run, they respect the "working_directory" +option if it is configured, meaning that the hook commands are run in that +directory. ### Order of execution @@ -104,7 +109,10 @@ configuration files. command hooks worked a little differently. In these older versions of borgmatic, you can specify `before_backup` hooks to perform preparation steps before running backups and specify `after_backup` hooks to perform cleanup steps -afterwards. Here's an example: +afterwards. These deprecated command hooks still work in version 2.0.0+, +although see below about a few semantic differences starting in that version. + +Here's an example of these deprecated hooks: ```yaml before_backup: @@ -129,6 +137,15 @@ instance, `before_prune` runs before a `prune` action for a repository, while Prior to version 1.8.0 Put these options in the `hooks:` section of your configuration. +New in version 2.0.0 An `after_*` +command hook runs even if an error occurs in the corresponding `before_*` hook +or between those two hooks. This allows you to perform cleanup steps that +correspond to `before_*` preparation commands—even when something goes wrong. + +New in version 2.0.0 When command +hooks run, they respect the "working_directory" option if it is configured, +meaning that the hook commands are run in that directory. + New in version 1.7.0 The `before_actions` and `after_actions` hooks run before/after all the actions (like `create`, `prune`, etc.) for each repository. These hooks are a good diff --git a/tests/unit/hooks/test_command.py b/tests/unit/hooks/test_command.py index f4925b36..ca6af167 100644 --- a/tests/unit/hooks/test_command.py +++ b/tests/unit/hooks/test_command.py @@ -190,11 +190,13 @@ def test_execute_hooks_invokes_each_hook_and_command(): output_log_level=LOGGING_ANSWER, shell=True, environment={}, + working_directory=None, ).once() module.execute_hooks( [{'before': 'create', 'run': ['foo']}, {'before': 'create', 'run': ['bar', 'baz']}], umask=None, + working_directory=None, dry_run=False, ) @@ -213,9 +215,35 @@ def test_execute_hooks_with_umask_sets_that_umask(): output_log_level=logging.ANSWER, shell=True, environment={}, + working_directory=None, ) - module.execute_hooks([{'before': 'create', 'run': ['foo']}], umask=77, dry_run=False) + module.execute_hooks( + [{'before': 'create', 'run': ['foo']}], umask=77, working_directory=None, dry_run=False + ) + + +def test_execute_hooks_with_working_directory_executes_command_with_it(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER + flexmock(module).should_receive('interpolate_context').replace_with( + lambda hook_description, command, context: command + ) + flexmock(module).should_receive('make_environment').and_return({}) + flexmock(module.borgmatic.execute).should_receive('execute_command').with_args( + ['foo'], + output_log_level=logging.ANSWER, + shell=True, + environment={}, + working_directory='/working', + ) + + module.execute_hooks( + [{'before': 'create', 'run': ['foo']}], + umask=None, + working_directory='/working', + dry_run=False, + ) def test_execute_hooks_with_dry_run_skips_commands(): @@ -227,11 +255,13 @@ def test_execute_hooks_with_dry_run_skips_commands(): flexmock(module).should_receive('make_environment').and_return({}) flexmock(module.borgmatic.execute).should_receive('execute_command').never() - module.execute_hooks([{'before': 'create', 'run': ['foo']}], umask=None, dry_run=True) + module.execute_hooks( + [{'before': 'create', 'run': ['foo']}], umask=None, working_directory=None, dry_run=True + ) def test_execute_hooks_with_empty_commands_does_not_raise(): - module.execute_hooks([], umask=None, dry_run=True) + module.execute_hooks([], umask=None, working_directory=None, dry_run=True) def test_execute_hooks_with_error_logs_as_error(): @@ -246,9 +276,12 @@ def test_execute_hooks_with_error_logs_as_error(): output_log_level=logging.ERROR, shell=True, environment={}, + working_directory=None, ).once() - module.execute_hooks([{'after': 'error', 'run': ['foo']}], umask=None, dry_run=False) + module.execute_hooks( + [{'after': 'error', 'run': ['foo']}], umask=None, working_directory=None, dry_run=False + ) def test_execute_hooks_with_before_or_after_raises(): @@ -265,6 +298,7 @@ def test_execute_hooks_with_before_or_after_raises(): {'erstwhile': 'create', 'run': ['bar', 'baz']}, ], umask=None, + working_directory=None, dry_run=False, ) @@ -283,11 +317,13 @@ def test_execute_hooks_without_commands_to_run_does_not_raise(): output_log_level=LOGGING_ANSWER, shell=True, environment={}, + working_directory=None, ).once() module.execute_hooks( [{'before': 'create', 'run': []}, {'before': 'create', 'run': ['foo', 'bar']}], umask=None, + working_directory=None, dry_run=False, ) @@ -315,6 +351,7 @@ def test_before_after_hooks_calls_command_hooks(): command_hooks=commands, before_after='action', umask=1234, + working_directory='/working', dry_run=False, hook_name='myhook', action_names=['create'], @@ -349,6 +386,7 @@ def test_before_after_hooks_with_before_error_runs_after_hook_and_raises(): command_hooks=commands, before_after='action', umask=1234, + working_directory='/working', dry_run=False, hook_name='myhook', action_names=['create'], @@ -382,6 +420,7 @@ def test_before_after_hooks_with_before_soft_failure_does_not_raise(): command_hooks=commands, before_after='action', umask=1234, + working_directory='/working', dry_run=False, hook_name='myhook', action_names=['create'], @@ -416,6 +455,7 @@ def test_before_after_hooks_with_after_error_raises(): command_hooks=commands, before_after='action', umask=1234, + working_directory='/working', dry_run=False, hook_name='myhook', action_names=['create'], @@ -449,6 +489,7 @@ def test_before_after_hooks_with_after_soft_failure_does_not_raise(): command_hooks=commands, before_after='action', umask=1234, + working_directory='/working', dry_run=False, hook_name='myhook', action_names=['create'], From 81a3a9957861413f58224d04c9d26e3b8d125db1 Mon Sep 17 00:00:00 2001 From: Benjamin Bock Date: Thu, 20 Mar 2025 22:49:04 +0100 Subject: [PATCH 137/226] Fix extracting from remote repositories with working_directory defined --- borgmatic/borg/extract.py | 4 +-- borgmatic/config/validate.py | 12 ++++++-- tests/unit/borg/test_extract.py | 4 +-- tests/unit/config/test_validate.py | 49 ++++++++++++++++++++++++++++-- 4 files changed, 59 insertions(+), 10 deletions(-) diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index 5097421c..52d7c35f 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -134,9 +134,7 @@ def extract_archive( # Make the repository path absolute so the destination directory used below via changing # the working directory doesn't prevent Borg from finding the repo. But also apply the # user's configured working directory (if any) to the repo path. - borgmatic.config.validate.normalize_repository_path( - os.path.join(working_directory or '', repository) - ), + borgmatic.config.validate.normalize_repository_path(repository, working_directory), archive, local_borg_version, ) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index c80cee4d..0334e301 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -138,16 +138,22 @@ def parse_configuration(config_filename, schema_filename, overrides=None, resolv return config, config_paths, logs -def normalize_repository_path(repository): +def normalize_repository_path(repository, base=None): ''' Given a repository path, return the absolute path of it (for local repositories). + Optionally, use a base path for resolving relative paths, e.g. to the configured working directory. ''' # A colon in the repository could mean that it's either a file:// URL or a remote repository. # If it's a remote repository, we don't want to normalize it. If it's a file:// URL, we do. if ':' not in repository: - return os.path.abspath(repository) + return ( + os.path.abspath(os.path.join(base, repository)) if base else os.path.abspath(repository) + ) elif repository.startswith('file://'): - return os.path.abspath(repository.partition('file://')[-1]) + local_path = repository.partition('file://')[-1] + return ( + os.path.abspath(os.path.join(base, local_path)) if base else os.path.abspath(local_path) + ) else: return repository diff --git a/tests/unit/borg/test_extract.py b/tests/unit/borg/test_extract.py index db244e54..de392f24 100644 --- a/tests/unit/borg/test_extract.py +++ b/tests/unit/borg/test_extract.py @@ -710,7 +710,7 @@ def test_extract_archive_uses_configured_working_directory_in_repo_path_and_dest ) flexmock(module.borgmatic.config.validate).should_receive( 'normalize_repository_path' - ).with_args('/working/dir/repo').and_return('/working/dir/repo').once() + ).with_args('repo', '/working/dir').and_return('/working/dir/repo').once() module.extract_archive( dry_run=False, @@ -733,7 +733,7 @@ def test_extract_archive_uses_configured_working_directory_in_repo_path_when_des ) flexmock(module.borgmatic.config.validate).should_receive( 'normalize_repository_path' - ).with_args('/working/dir/repo').and_return('/working/dir/repo').once() + ).with_args('repo', '/working/dir').and_return('/working/dir/repo').once() module.extract_archive( dry_run=False, diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index 8490bd07..04b5bb40 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -94,13 +94,40 @@ def test_normalize_repository_path_passes_through_remote_repository(): module.normalize_repository_path(repository) == repository +def test_normalize_repository_path_passes_through_remote_repository_with_base_dir(): + repository = 'example.org:test.borg' + + flexmock(module.os.path).should_receive('abspath').never() + module.normalize_repository_path(repository, '/working') == repository + + def test_normalize_repository_path_passes_through_file_repository(): repository = 'file:///foo/bar/test.borg' - flexmock(module.os.path).should_receive('abspath').and_return('/foo/bar/test.borg') + flexmock(module.os.path).should_receive('abspath').with_args('/foo/bar/test.borg').and_return( + '/foo/bar/test.borg' + ) module.normalize_repository_path(repository) == '/foo/bar/test.borg' +def test_normalize_repository_path_passes_through_absolute_file_repository_with_base_dir(): + repository = 'file:///foo/bar/test.borg' + flexmock(module.os.path).should_receive('abspath').with_args('/foo/bar/test.borg').and_return( + '/foo/bar/test.borg' + ) + + module.normalize_repository_path(repository, '/working') == '/foo/bar/test.borg' + + +def test_normalize_repository_path_resolves_relative_file_repository_with_base_dir(): + repository = 'file://foo/bar/test.borg' + flexmock(module.os.path).should_receive('abspath').with_args( + '/working/foo/bar/test.borg' + ).and_return('/working/foo/bar/test.borg') + + module.normalize_repository_path(repository, '/working') == '/working/foo/bar/test.borg' + + def test_normalize_repository_path_passes_through_absolute_repository(): repository = '/foo/bar/test.borg' flexmock(module.os.path).should_receive('abspath').and_return(repository) @@ -108,14 +135,32 @@ def test_normalize_repository_path_passes_through_absolute_repository(): module.normalize_repository_path(repository) == repository +def test_normalize_repository_path_passes_through_absolute_repository_with_base_dir(): + repository = '/foo/bar/test.borg' + flexmock(module.os.path).should_receive('abspath').and_return(repository) + + module.normalize_repository_path(repository, '/working') == repository + + def test_normalize_repository_path_resolves_relative_repository(): repository = 'test.borg' absolute = '/foo/bar/test.borg' - flexmock(module.os.path).should_receive('abspath').and_return(absolute) + flexmock(module.os.path).should_receive('abspath').with_args(repository).and_return(absolute) module.normalize_repository_path(repository) == absolute +def test_normalize_repository_path_resolves_relative_repository_with_base_dir(): + repository = 'test.borg' + base = '/working' + absolute = '/working/test.borg' + flexmock(module.os.path).should_receive('abspath').with_args('/working/test.borg').and_return( + absolute + ) + + module.normalize_repository_path(repository, base) == absolute + + @pytest.mark.parametrize( 'first,second,expected_result', ( From a16d138afcb94ad11f52204710c3a6625549493c Mon Sep 17 00:00:00 2001 From: Benjamin Bock Date: Fri, 21 Mar 2025 21:58:02 +0100 Subject: [PATCH 138/226] Crontabs aren't executable --- docs/how-to/set-up-backups.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 3965e12e..c90b9136 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -311,7 +311,6 @@ Then, from the directory where you downloaded it: ```bash sudo mv borgmatic /etc/cron.d/borgmatic -sudo chmod +x /etc/cron.d/borgmatic ``` If borgmatic is installed at a different location than From 201469e2c2b8afdc95379320056310f055b17d39 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 21 Mar 2025 14:26:01 -0700 Subject: [PATCH 139/226] Add "key import" action to NEWS (#345). --- NEWS | 1 + borgmatic/borg/import_key.py | 1 - borgmatic/commands/borgmatic.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index f7d06b2c..89d90393 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,5 @@ 2.0.0.dev0 + * #345: Add a "key import" action to import a repository key from backup. * #790, #821: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more flexible "commands:". See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ diff --git a/borgmatic/borg/import_key.py b/borgmatic/borg/import_key.py index 7f011ebc..9e902778 100644 --- a/borgmatic/borg/import_key.py +++ b/borgmatic/borg/import_key.py @@ -27,7 +27,6 @@ def import_key( Raise ValueError if the path is given and it does not exist. ''' - borgmatic.logger.add_custom_log_levels() umask = config.get('umask', None) lock_wait = config.get('lock_wait', None) working_directory = borgmatic.config.paths.get_working_directory(config) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 9c450936..bd1ec1e6 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -20,8 +20,8 @@ import borgmatic.actions.create import borgmatic.actions.delete import borgmatic.actions.export_key import borgmatic.actions.export_tar -import borgmatic.actions.import_key import borgmatic.actions.extract +import borgmatic.actions.import_key import borgmatic.actions.info import borgmatic.actions.list import borgmatic.actions.mount From 514ade6609af1ae56ea9e6104c22794b68bee496 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 21 Mar 2025 14:27:40 -0700 Subject: [PATCH 140/226] Fix inconsistent quotes in one documentation file (#790). --- docs/how-to/add-preparation-and-cleanup-steps-to-backups.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index 977e87f1..7637fd72 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -72,7 +72,7 @@ steps that correspond to `before` preparation commands—even when something goe wrong. This is a departure from the way that the deprecated `after_*` hooks worked in borgmatic prior to version 2.0.0. -Additionally, when command hooks run, they respect the "working_directory" +Additionally, when command hooks run, they respect the `working_directory` option if it is configured, meaning that the hook commands are run in that directory. @@ -143,7 +143,7 @@ or between those two hooks. This allows you to perform cleanup steps that correspond to `before_*` preparation commands—even when something goes wrong. New in version 2.0.0 When command -hooks run, they respect the "working_directory" option if it is configured, +hooks run, they respect the `working_directory` option if it is configured, meaning that the hook commands are run in that directory. New in version 1.7.0 The From 524ec6b3cbdabf198790c44c7904e4c761836336 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 21 Mar 2025 15:43:05 -0700 Subject: [PATCH 141/226] Add "extract" action fix to NEWS (#1037). --- NEWS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS b/NEWS index 89d90393..a7da35bc 100644 --- a/NEWS +++ b/NEWS @@ -12,6 +12,8 @@ * #1010: When using Borg 2, don't pass the "--stats" flag to "borg prune". * #1020: Document a database use case involving a temporary database client container: https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers + * #1037: Fix an error with the "extract" action when both a remote repository and a + "working_directory" are used. 1.9.14 * #409: With the PagerDuty monitoring hook, send borgmatic logs to PagerDuty so they show up in the From 976fb8f343429715cf3ec0a353f64277df3e7ae3 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 21 Mar 2025 22:44:49 -0700 Subject: [PATCH 142/226] Add "compact_threshold" option, overridden by "compact --threshold" flag (#303). --- borgmatic/actions/compact.py | 2 +- borgmatic/config/schema.yaml | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index 1a0cea3d..7386d230 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -39,7 +39,7 @@ def run_compact( remote_path=remote_path, progress=compact_arguments.progress or config.get('progress'), cleanup_commits=compact_arguments.cleanup_commits, - threshold=compact_arguments.threshold, + threshold=compact_arguments.threshold or config.get('compact_threshold'), ) else: # pragma: nocover logger.info('Skipping compact (only available/needed in Borg 1.2+)') diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 86cc5255..a7c9fc4c 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -565,6 +565,12 @@ properties: not specified, borgmatic defaults to matching archives based on the archive_name_format (see above). example: sourcehostname + compact_threshold: + type: integer + description: | + Minimum saved space percentage threshold for compacting a segment, + defaults to 10. + example: 20 checks: type: array items: From a750d58a2d972e2255c9e014e27a3800c5270a62 Mon Sep 17 00:00:00 2001 From: Vandal <86826719+VandalByte@users.noreply.github.com> Date: Sat, 22 Mar 2025 21:18:28 +0530 Subject: [PATCH 143/226] add recreate action --- borgmatic/actions/recreate.py | 46 +++++++++++++++++++++++++ borgmatic/borg/recreate.py | 61 ++++++++------------------------- borgmatic/commands/arguments.py | 21 ++++++++++++ borgmatic/commands/borgmatic.py | 11 ++++++ 4 files changed, 93 insertions(+), 46 deletions(-) diff --git a/borgmatic/actions/recreate.py b/borgmatic/actions/recreate.py index e69de29b..acac0d44 100644 --- a/borgmatic/actions/recreate.py +++ b/borgmatic/actions/recreate.py @@ -0,0 +1,46 @@ +import logging + +import borgmatic.borg.recreate +import borgmatic.config.validate + +logger = logging.getLogger(__name__) + + +def run_recreate( + repository, + config, + local_borg_version, + recreate_arguments, + global_arguments, + local_path, + remote_path, +): + ''' + Run the "recreate" action for the given repository. + ''' + if recreate_arguments.repository is None or borgmatic.config.validate.repositories_match( + repository, recreate_arguments.repository + ): + if recreate_arguments.archive: + logger.info(f'Recreating archive {recreate_arguments.archive}') + else: + logger.info('Recreating repository') + + borgmatic.borg.recreate.recreate_archive( + repository['path'], + borgmatic.borg.repo_list.resolve_archive_name( + repository['path'], + recreate_arguments.archive, + config, + local_borg_version, + global_arguments, + local_path, + remote_path, + ), + config, + local_borg_version, + recreate_arguments, + global_arguments, + local_path=local_path, + remote_path=remote_path, + ) diff --git a/borgmatic/borg/recreate.py b/borgmatic/borg/recreate.py index a9c6ff56..a1bd60aa 100644 --- a/borgmatic/borg/recreate.py +++ b/borgmatic/borg/recreate.py @@ -9,7 +9,7 @@ from borgmatic.borg.flags import make_flags_from_arguments, make_repository_arch logger = logging.getLogger(__name__) -def make_recreate_command( +def recreate_archive( repository, archive, config, @@ -24,61 +24,30 @@ def make_recreate_command( the local Borg version string, an argparse.Namespace of recreate arguments, an argparse.Namespace of global arguments, optional local and remote Borg paths. - Returns the recreate command as a tuple of strings ready for execution. + Executes the recreate command with the given arguments. ''' - verbosity_flags = (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + ( - ('--info',) if logger.isEnabledFor(logging.INFO) else () - ) + lock_wait = config.get('lock_wait', None) - # handle both the recreate and global arguments - recreate_flags = make_flags_from_arguments( - recreate_arguments, excludes=('repository', 'archive') - ) - global_flags = make_flags_from_arguments(global_arguments) - - repo_archive_flags = make_repository_archive_flags(repository, archive, local_borg_version) + repo_archive_arg = make_repository_archive_flags(repository, archive, local_borg_version) exclude_flags = make_exclude_flags(config) - return ( + recreate_cmd = ( (local_path, 'recreate') - + repo_archive_flags - + verbosity_flags - + global_flags - + recreate_flags + + (('--remote-path', remote_path) if remote_path else ()) + + repo_archive_arg + + (('--log-json',) if global_arguments.log_json else ()) + + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + + (('--debug', '--show-rc', '--list') if logger.isEnabledFor(logging.DEBUG) else ()) + exclude_flags ) - -def recreate_archive( - repository, - archive, - config, - local_borg_version, - recreate_arguments, - global_arguments, - local_path='borg', - remote_path=None, -): - ''' - Given a local or remote repository path, an archive name, a configuration dict, - the local Borg version string, an argparse.Namespace of recreate arguments, - an argparse.Namespace of global arguments, optional local and remote Borg paths. - - Executes the recreate command with the given arguments. - ''' - command = make_recreate_command( - repository, - archive, - config, - local_borg_version, - recreate_arguments, - global_arguments, - local_path, - remote_path, - ) + if global_arguments.dry_run: + logger.info('Skipping the archive recreation (dry run)') + return borgmatic.execute.execute_command( - command, + recreate_cmd, output_log_level=logging.ANSWER, environment=borgmatic.borg.environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 56842ea8..4018547e 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -27,6 +27,7 @@ ACTION_ALIASES = { 'break-lock': [], 'key': [], 'borg': [], + 'recreate': [], } @@ -1545,6 +1546,26 @@ def make_parsers(): ) borg_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') + recreate_parser = action_parsers.add_parser( + 'recreate', + aliases=ACTION_ALIASES['recreate'], + help='Recreate an archive in a repository', + description='Recreate an archive in a repository', + add_help=False, + ) + recreate_group = recreate_parser.add_argument_group('recreate arguments') + recreate_group.add_argument( + '--repository', + help='Path of the repository containing the archive', + ) + recreate_group.add_argument( + '--archive', + help='Name of the archive to recreate', + ) + recreate_group.add_argument( + '-h', '--help', action='help', help='Show this help message and exit' + ) + return global_parser, action_parsers, global_plus_action_parser diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index bd1ec1e6..467673fe 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -17,6 +17,7 @@ import borgmatic.actions.config.bootstrap import borgmatic.actions.config.generate import borgmatic.actions.config.validate import borgmatic.actions.create +import borgmatic.actions.recreate import borgmatic.actions.delete import borgmatic.actions.export_key import borgmatic.actions.export_tar @@ -397,6 +398,16 @@ def run_actions( local_path, remote_path, ) + elif action_name == 'recreate' and action_name not in skip_actions: + borgmatic.actions.recreate.run_recreate( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) elif action_name == 'prune' and action_name not in skip_actions: borgmatic.actions.prune.run_prune( config_filename, From cc1442146098d1b9bc63633dd324454abd9a22bf Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Mar 2025 13:58:42 -0700 Subject: [PATCH 144/226] Fix list examples in generated configuration. --- borgmatic/config/generate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index c40d9f9b..bf3b3486 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -38,6 +38,7 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i if schema_type == 'array' or (isinstance(schema_type, list) and 'array' in schema_type): config = ruamel.yaml.comments.CommentedSeq( + example if schema['items'].get('type') in SCALAR_SCHEMA_TYPES else [ schema_to_sample_configuration( schema['items'], source_config, level, parent_is_sequence=True @@ -53,7 +54,7 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i [ ( field_name, - sub_schema.get('example') if field_name == 'source_directories' else schema_to_sample_configuration( + schema_to_sample_configuration( sub_schema, (source_config or {}).get(field_name, {}), level + 1 ), ) From f8eda92379bca390936b233c19e134a828c46eef Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Mar 2025 14:01:39 -0700 Subject: [PATCH 145/226] Code formatting (#303). --- borgmatic/borg/check.py | 4 +--- borgmatic/borg/delete.py | 4 +++- borgmatic/borg/transfer.py | 4 +++- borgmatic/config/generate.py | 9 ++++++--- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 98c48946..13f4ffca 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -180,9 +180,7 @@ def check_archives( full_command, # The Borg repair option triggers an interactive prompt, which won't work when output is # captured. And progress messes with the terminal directly. - output_file=( - DO_NOT_CAPTURE if check_arguments.repair or progress else None - ), + output_file=(DO_NOT_CAPTURE if check_arguments.repair or progress else None), environment=environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, diff --git a/borgmatic/borg/delete.py b/borgmatic/borg/delete.py index d5b02335..a9c3478c 100644 --- a/borgmatic/borg/delete.py +++ b/borgmatic/borg/delete.py @@ -34,7 +34,9 @@ def make_delete_command( + borgmatic.borg.flags.make_flags('umask', config.get('umask')) + borgmatic.borg.flags.make_flags('log-json', global_arguments.log_json) + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait')) - + borgmatic.borg.flags.make_flags('list', delete_arguments.list_archives or config.get('list')) + + borgmatic.borg.flags.make_flags( + 'list', delete_arguments.list_archives or config.get('list') + ) + ( (('--force',) + (('--force',) if delete_arguments.force >= 2 else ())) if delete_arguments.force diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index 32dc48b3..ca90063e 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -56,7 +56,9 @@ def transfer_archives( return execute_command( full_command, output_log_level=logging.ANSWER, - output_file=DO_NOT_CAPTURE if (transfer_arguments.progress or config.get('progress')) else None, + output_file=( + DO_NOT_CAPTURE if (transfer_arguments.progress or config.get('progress')) else None + ), environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index bf3b3486..53f20133 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -38,8 +38,9 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i if schema_type == 'array' or (isinstance(schema_type, list) and 'array' in schema_type): config = ruamel.yaml.comments.CommentedSeq( - example if schema['items'].get('type') in SCALAR_SCHEMA_TYPES else - [ + example + if schema['items'].get('type') in SCALAR_SCHEMA_TYPES + else [ schema_to_sample_configuration( schema['items'], source_config, level, parent_is_sequence=True ) @@ -65,7 +66,9 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i add_comments_to_configuration_object( config, schema, source_config, indent=indent, skip_first=parent_is_sequence ) - elif isinstance(schema_type, list) and all(element_schema_type in SCALAR_SCHEMA_TYPES for element_schema_type in schema_type): + elif isinstance(schema_type, list) and all( + element_schema_type in SCALAR_SCHEMA_TYPES for element_schema_type in schema_type + ): return example elif schema_type in SCALAR_SCHEMA_TYPES: return example From dc9da3832d1c661f06407e9c8dce0c5cf20030ab Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Mar 2025 14:03:44 -0700 Subject: [PATCH 146/226] Bold "not yet released" in docs to prevent confusion (#303). --- docs/how-to/add-preparation-and-cleanup-steps-to-backups.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index 7637fd72..3aecf478 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -17,8 +17,8 @@ points as it runs. feature](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/) instead.) -New in version 2.0.0 (not yet -released) Command hooks are now configured via a list of `commands:` in +New in version 2.0.0 (**not yet +released**) Command hooks are now configured via a list of `commands:` in your borgmatic configuration file. For example: ```yaml From f222bf2c1a5fb6d8a54d73672503259599d29b32 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 22 Mar 2025 22:52:23 -0700 Subject: [PATCH 147/226] Organizational refactoring (#303). --- borgmatic/commands/arguments.py | 152 ++++++++++++++++++-------------- borgmatic/config/arguments.py | 8 +- borgmatic/config/schema.py | 7 +- 3 files changed, 98 insertions(+), 69 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index f00f49bb..32249f8f 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -291,6 +291,91 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse OMITTED_FLAG_NAMES = {'match_archives', 'progress', 'stats', 'list'} +def make_argument_description(schema, flag_name): + ''' + Given a configuration schema dict and a flag name for it, extend the schema's description with + an example or additional information as appropriate based on its type. Return the updated + description for use in a command-line argument. + ''' + description = schema.get('description') + schema_type = schema.get('type') + + if not description: + return None + + if schema_type == 'array': + example_buffer = io.StringIO() + yaml = ruamel.yaml.YAML(typ='safe') + yaml.default_flow_style = True + yaml.dump(schema.get('example'), example_buffer) + + description += f' Example value: "{example_buffer.getvalue().strip()}"' + + if '[0]' in flag_name: + description += ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).' + + description = description.replace('%', '%%') + + +def add_array_element_arguments_from_schema(arguments_group, schema, unparsed_arguments, flag_name): + ''' + Given an argparse._ArgumentGroup instance, a configuration schema dict, a sequence of unparsed + argument strings, and a dotted flag name, convert the schema into corresponding command-line + array element flags that correspond to the given unparsed arguments. + + Here's the background. We want to support flags that can have arbitrary indices like: + + --foo.bar[1].baz + + But argparse doesn't support that natively because the index can be an arbitrary number. We + won't let that stop us though, will we? + + If the current flag name has an array component in it (e.g. a name with "[0]"), then make a + pattern that would match the flag name regardless of the number that's in it. The idea is that + we want to look for unparsed arguments that appear like the flag name, but instead of "[0]" they + have, say, "[1]" or "[123]". + + Next, we check each unparsed argument against that pattern. If one of them matches, add an + argument flag for it to the argument parser group. Example: + + Let's say flag_name is: + + --foo.bar[0].baz + + ... then the regular expression pattern will be: + + ^--foo\.bar\[\d+\]\.baz + + ... and, if that matches an unparsed argument of: + + --foo.bar[1].baz + + ... then an argument flag will get added equal to that unparsed argument. And the unparsed + argument will match it when parsing is performed! In this manner, we're using the actual user + CLI input to inform what exact flags we support! + ''' + if '[0]' not in flag_name or '--help' in unparsed_arguments: + return + + pattern = re.compile(f'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.")}$') + existing_flags = set( + itertools.chain( + *(group_action.option_strings for group_action in arguments_group._group_actions) + ) + ) + + for unparsed in unparsed_arguments: + unparsed_flag_name = unparsed.split('=', 1)[0] + + if pattern.match(unparsed_flag_name) and unparsed_flag_name not in existing_flags: + arguments_group.add_argument( + unparsed_flag_name, + type=argument_type, + metavar=metavar, + help=description, + ) + + def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names=None): ''' Given an argparse._ArgumentGroup instance, a configuration schema dict, and a sequence of @@ -362,28 +447,14 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names ) flag_name = '.'.join(names) - description = schema.get('description') metavar = names[-1].upper() - if description: - if schema_type == 'array': - example_buffer = io.StringIO() - yaml = ruamel.yaml.YAML(typ='safe') - yaml.default_flow_style = True - yaml.dump(schema.get('example'), example_buffer) - - description += f' Example value: "{example_buffer.getvalue().strip()}"' - - if '[0]' in flag_name: - description += ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).' - - description = description.replace('%', '%%') - - # These options already have corresponding flags on individual actions (like "create + # Certain options already have corresponding flags on individual actions (like "create # --progress"), so don't bother adding them to the global flags. if flag_name in OMITTED_FLAG_NAMES: return + description = make_argument_description(schema, flag_name) argument_type = borgmatic.config.schema.parse_type(schema_type) full_flag_name = f"--{flag_name.replace('_', '-')}" @@ -404,54 +475,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names help=description, ) - # We want to support flags that can have arbitrary indices like: - # - # --foo.bar[1].baz - # - # But argparse doesn't support that natively because the index can be an arbitrary number. We - # won't let that stop us though, will we? So, if the current flag name has an array component in - # it (e.g. a name with "[0]"), then make a pattern that would match the flag name regardless of - # the number that's in it. The idea is that we want to look for unparsed arguments that appear - # like the flag name, but instead of "[0]" they have, say, "[1]" or "[123]". - # - # Next, we check each unparsed argument against that pattern. If one of them matches, add an - # argument flag for it to the argument parser group. Example: - # - # Let's say flag_name is: - # - # --foo.bar[0].baz - # - # ... then the regular expression pattern will be: - # - # ^--foo\.bar\[\d+\]\.baz - # - # ... and, if that matches an unparsed argument of: - # - # --foo.bar[1].baz - # - # ... then an argument flag will get added equal to that unparsed argument. And the unparsed - # argument will match it when parsing is performed! In this manner, we're using the actual user - # CLI input to inform what exact flags we support! - if '[0]' not in flag_name or '--help' in unparsed_arguments: - return - - pattern = re.compile(f'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.")}$') - existing_flags = set( - itertools.chain( - *(group_action.option_strings for group_action in arguments_group._group_actions) - ) - ) - - for unparsed in unparsed_arguments: - unparsed_flag_name = unparsed.split('=', 1)[0] - - if pattern.match(unparsed_flag_name) and unparsed_flag_name not in existing_flags: - arguments_group.add_argument( - unparsed_flag_name, - type=argument_type, - metavar=metavar, - help=description, - ) + add_array_element_arguments_from_schema(arguments_group, schema, unparsed_arguments, flag_name) def make_parsers(schema, unparsed_arguments): diff --git a/borgmatic/config/arguments.py b/borgmatic/config/arguments.py index 9aa83998..8f7023cd 100644 --- a/borgmatic/config/arguments.py +++ b/borgmatic/config/arguments.py @@ -14,13 +14,13 @@ def set_values(config, keys, value): ('foo', 'bar', 'baz') - This looks up "foo" in the given configuration. And within that value, it looks up "bar". And - then within that value, it looks up "baz" and sets it to the given value. Another example: + This looks up "foo" in the given configuration dict. And within that, it looks up "bar". And + then within that, it looks up "baz" and sets it to the given value. Another example: ('mylist[0]', 'foo') - This looks for the zeroth element of "mylist" in the given configuration. And within that value, - it looks up "foo" and sets it to the given value. + This looks for the zeroth element of "mylist" in the given configuration. And within that, it + looks up "foo" and sets it to the given value. ''' if not keys: return diff --git a/borgmatic/config/schema.py b/borgmatic/config/schema.py index 118e9437..4442010f 100644 --- a/borgmatic/config/schema.py +++ b/borgmatic/config/schema.py @@ -5,7 +5,7 @@ import itertools def get_properties(schema): ''' Given a schema dict, return its properties. But if it's got sub-schemas with multiple different - potential properties, returned their merged properties instead (interleaved so the first + potential properties, return their merged properties instead (interleaved so the first properties of each sub-schema come first). The idea is that the user should see all possible options even if they're not all possible together. ''' @@ -24,6 +24,11 @@ def get_properties(schema): def parse_type(schema_type): + ''' + Given a schema type as a string, return the corresponding Python type. + + Raise ValueError if the schema type is unknown. + ''' try: return { 'string': str, From 57721937a3729ce8e1b6cbb1edcbbf85eb2b8a7a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 23 Mar 2025 11:24:36 -0700 Subject: [PATCH 148/226] Factor out schema type comparion in config generation and get several tests passing (#303). --- borgmatic/commands/arguments.py | 1 + borgmatic/config/generate.py | 12 ++++-------- borgmatic/config/schema.py | 22 ++++++++++++++++++++++ tests/unit/config/test_generate.py | 28 ++++++++++++++++++++-------- 4 files changed, 47 insertions(+), 16 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 32249f8f..2d285f50 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -297,6 +297,7 @@ def make_argument_description(schema, flag_name): an example or additional information as appropriate based on its type. Return the updated description for use in a command-line argument. ''' + # FIXME: Argument descriptions are apparently broken right now. description = schema.get('description') schema_type = schema.get('type') diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index 53f20133..58360ad0 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -36,10 +36,10 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i schema_type = schema.get('type') example = schema.get('example') - if schema_type == 'array' or (isinstance(schema_type, list) and 'array' in schema_type): + if borgmatic.config.schema.compare_types(schema_type, {'array'}): config = ruamel.yaml.comments.CommentedSeq( example - if schema['items'].get('type') in SCALAR_SCHEMA_TYPES + if borgmatic.config.schema.compare_types(schema['items'].get('type'), SCALAR_SCHEMA_TYPES) else [ schema_to_sample_configuration( schema['items'], source_config, level, parent_is_sequence=True @@ -47,7 +47,7 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i ] ) add_comments_to_configuration_sequence(config, schema, indent=(level * INDENT)) - elif schema_type == 'object' or (isinstance(schema_type, list) and 'object' in schema_type): + elif borgmatic.config.schema.compare_types(schema_type, {'object'}): if source_config and isinstance(source_config, list) and isinstance(source_config[0], dict): source_config = dict(collections.ChainMap(*source_config)) @@ -66,11 +66,7 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i add_comments_to_configuration_object( config, schema, source_config, indent=indent, skip_first=parent_is_sequence ) - elif isinstance(schema_type, list) and all( - element_schema_type in SCALAR_SCHEMA_TYPES for element_schema_type in schema_type - ): - return example - elif schema_type in SCALAR_SCHEMA_TYPES: + elif borgmatic.config.schema.compare_types(schema_type, SCALAR_SCHEMA_TYPES, match=all): return example else: raise ValueError(f'Schema at level {level} is unsupported: {schema}') diff --git a/borgmatic/config/schema.py b/borgmatic/config/schema.py index 4442010f..d4698c3a 100644 --- a/borgmatic/config/schema.py +++ b/borgmatic/config/schema.py @@ -39,3 +39,25 @@ def parse_type(schema_type): }[schema_type] except KeyError: raise ValueError(f'Unknown type in configuration schema: {schema_type}') + + +def compare_types(schema_type, target_types, match=any): + ''' + Given a schema type as a string or a list of strings (representing multiple types) and a set of + target type strings, return whether every schema type is in the set of target types. + + If the schema type is a list of strings, use the given match function (such as any or all) to + compare elements. + ''' + if isinstance(schema_type, list): + if match( + element_schema_type in target_types for element_schema_type in schema_type + ): + return True + + return False + + if schema_type in target_types: + return True + + return False diff --git a/tests/unit/config/test_generate.py b/tests/unit/config/test_generate.py index abb520fb..9c07001d 100644 --- a/tests/unit/config/test_generate.py +++ b/tests/unit/config/test_generate.py @@ -2,6 +2,7 @@ import pytest from flexmock import flexmock from borgmatic.config import generate as module +import borgmatic.config.schema def test_schema_to_sample_configuration_generates_config_map_with_examples(): @@ -9,13 +10,16 @@ def test_schema_to_sample_configuration_generates_config_map_with_examples(): 'type': 'object', 'properties': dict( [ - ('field1', {'example': 'Example 1'}), - ('field2', {'example': 'Example 2'}), - ('field3', {'example': 'Example 3'}), + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ('field3', {'type': 'string', 'example': 'Example 3'}), ] ), } - flexmock(module).should_receive('get_properties').and_return(schema['properties']) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').and_return(False) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('object', {'object'}).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('string', module.SCALAR_SCHEMA_TYPES, match=all).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return(schema['properties']) flexmock(module.ruamel.yaml.comments).should_receive('CommentedMap').replace_with(dict) flexmock(module).should_receive('add_comments_to_configuration_object') @@ -46,11 +50,15 @@ def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_e 'items': { 'type': 'object', 'properties': dict( - [('field1', {'example': 'Example 1'}), ('field2', {'example': 'Example 2'})] + [('field1', {'type': 'string', 'example': 'Example 1'}), ('field2', {'type': 'string', 'example': 'Example 2'})] ), }, } - flexmock(module).should_receive('get_properties').and_return(schema['items']['properties']) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').and_return(False) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('array', {'array'}).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('object', {'object'}).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('string', module.SCALAR_SCHEMA_TYPES, match=all).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return(schema['items']['properties']) flexmock(module.ruamel.yaml.comments).should_receive('CommentedSeq').replace_with(list) flexmock(module).should_receive('add_comments_to_configuration_sequence') flexmock(module).should_receive('add_comments_to_configuration_object') @@ -66,11 +74,15 @@ def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_m 'items': { 'type': ['object', 'null'], 'properties': dict( - [('field1', {'example': 'Example 1'}), ('field2', {'example': 'Example 2'})] + [('field1', {'type': 'string', 'example': 'Example 1'}), ('field2', {'type': 'string', 'example': 'Example 2'})] ), }, } - flexmock(module).should_receive('get_properties').and_return(schema['items']['properties']) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').and_return(False) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('array', {'array'}).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args(['object', 'null'], {'object'}).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('string', module.SCALAR_SCHEMA_TYPES, match=all).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return(schema['items']['properties']) flexmock(module.ruamel.yaml.comments).should_receive('CommentedSeq').replace_with(list) flexmock(module).should_receive('add_comments_to_configuration_sequence') flexmock(module).should_receive('add_comments_to_configuration_object') From ba75958a2f043f491ffc57673951bf71bc8b197a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 23 Mar 2025 11:26:49 -0700 Subject: [PATCH 149/226] Fix missing argument descriptions (#303). --- borgmatic/commands/arguments.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 2d285f50..289d163f 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -297,7 +297,6 @@ def make_argument_description(schema, flag_name): an example or additional information as appropriate based on its type. Return the updated description for use in a command-line argument. ''' - # FIXME: Argument descriptions are apparently broken right now. description = schema.get('description') schema_type = schema.get('type') @@ -317,6 +316,8 @@ def make_argument_description(schema, flag_name): description = description.replace('%', '%%') + return description + def add_array_element_arguments_from_schema(arguments_group, schema, unparsed_arguments, flag_name): ''' From 9f7c71265e82ec1d470f9a6e8748868e422095f5 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 23 Mar 2025 16:32:31 -0700 Subject: [PATCH 150/226] Add Bash completion for completing flags like "--foo[3].bar". --- borgmatic/commands/completion/bash.py | 12 ++++++++++-- borgmatic/commands/completion/fish.py | 23 +++++++++++++++-------- borgmatic/commands/completion/flag.py | 16 ++++++++++++++++ 3 files changed, 41 insertions(+), 10 deletions(-) create mode 100644 borgmatic/commands/completion/flag.py diff --git a/borgmatic/commands/completion/bash.py b/borgmatic/commands/completion/bash.py index 7bf28a42..6eb5e92d 100644 --- a/borgmatic/commands/completion/bash.py +++ b/borgmatic/commands/completion/bash.py @@ -1,5 +1,6 @@ import borgmatic.commands.arguments import borgmatic.commands.completion.actions +import borgmatic.commands.completion.flag def parser_flags(parser): @@ -7,7 +8,11 @@ def parser_flags(parser): Given an argparse.ArgumentParser instance, return its argument flags in a space-separated string. ''' - return ' '.join(option for action in parser._actions for option in action.option_strings) + return ' '.join( + flag_variant + for action in parser._actions for flag_name in action.option_strings + for flag_variant in borgmatic.commands.completion.flag.variants(flag_name) + ) def bash_completion(): @@ -19,7 +24,10 @@ def bash_completion(): unused_global_parser, action_parsers, global_plus_action_parser, - ) = borgmatic.commands.arguments.make_parsers() + ) = borgmatic.commands.arguments.make_parsers( + schema=borgmatic.config.validate.load_schema(borgmatic.config.validate.schema_filename()), + unparsed_arguments=(), + ) global_flags = parser_flags(global_plus_action_parser) # Avert your eyes. diff --git a/borgmatic/commands/completion/fish.py b/borgmatic/commands/completion/fish.py index edca0226..31b83e9c 100644 --- a/borgmatic/commands/completion/fish.py +++ b/borgmatic/commands/completion/fish.py @@ -4,6 +4,7 @@ from textwrap import dedent import borgmatic.commands.arguments import borgmatic.commands.completion.actions +import borgmatic.config.validate def has_file_options(action: Action): @@ -26,9 +27,11 @@ def has_choice_options(action: Action): def has_unknown_required_param_options(action: Action): ''' A catch-all for options that take a required parameter, but we don't know what the parameter is. - This should be used last. These are actions that take something like a glob, a list of numbers, or a string. + This should be used last. These are actions that take something like a glob, a list of numbers, + or a string. - Actions that match this pattern should not show the normal arguments, because those are unlikely to be valid. + Actions that match this pattern should not show the normal arguments, because those are unlikely + to be valid. ''' return ( action.required is True @@ -52,9 +55,9 @@ def has_exact_options(action: Action): def exact_options_completion(action: Action): ''' - Given an argparse.Action instance, return a completion invocation that forces file completions, options completion, - or just that some value follow the action, if the action takes such an argument and was the last action on the - command line prior to the cursor. + Given an argparse.Action instance, return a completion invocation that forces file completions, + options completion, or just that some value follow the action, if the action takes such an + argument and was the last action on the command line prior to the cursor. Otherwise, return an empty string. ''' @@ -80,8 +83,9 @@ def exact_options_completion(action: Action): def dedent_strip_as_tuple(string: str): ''' - Dedent a string, then strip it to avoid requiring your first line to have content, then return a tuple of the string. - Makes it easier to write multiline strings for completions when you join them with a tuple. + Dedent a string, then strip it to avoid requiring your first line to have content, then return a + tuple of the string. Makes it easier to write multiline strings for completions when you join + them with a tuple. ''' return (dedent(string).strip('\n'),) @@ -95,7 +99,10 @@ def fish_completion(): unused_global_parser, action_parsers, global_plus_action_parser, - ) = borgmatic.commands.arguments.make_parsers() + ) = borgmatic.commands.arguments.make_parsers( + schema=borgmatic.config.validate.load_schema(borgmatic.config.validate.schema_filename()), + unparsed_arguments=(), + ) all_action_parsers = ' '.join(action for action in action_parsers.choices.keys()) diff --git a/borgmatic/commands/completion/flag.py b/borgmatic/commands/completion/flag.py new file mode 100644 index 00000000..67cf0712 --- /dev/null +++ b/borgmatic/commands/completion/flag.py @@ -0,0 +1,16 @@ +def variants(flag_name): + ''' + Given an flag name as a string, yield it and any variations that should be complete-able as + well. For instance, for a string like "--foo[0].bar", yield "--foo[0].bar", "--foo[1].bar", ..., + "--foo[9].bar". + ''' + if '[0]' in flag_name: + for index in range(0, 10): + yield flag_name.replace('[0]', f'[{index}]') + + return + + yield flag_name + + + From 423627e67bf81873d65fdc9c1388ed1e54dcb1c6 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 23 Mar 2025 17:00:04 -0700 Subject: [PATCH 151/226] Get existing unit/integration tests passing (#303). --- borgmatic/commands/arguments.py | 52 ++--- borgmatic/commands/completion/bash.py | 4 +- borgmatic/commands/completion/flag.py | 3 - borgmatic/config/generate.py | 4 +- borgmatic/config/schema.py | 4 +- borgmatic/config/validate.py | 12 ++ tests/integration/borg/test_commands.py | 12 +- .../commands/completion/test_actions.py | 11 +- tests/integration/commands/test_arguments.py | 180 ++++++++++-------- tests/integration/config/test_generate.py | 24 +-- tests/integration/config/test_validate.py | 43 +++-- tests/unit/commands/test_borgmatic.py | 9 +- tests/unit/config/test_generate.py | 54 ++++-- 13 files changed, 252 insertions(+), 160 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 289d163f..97dc3361 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -69,9 +69,9 @@ def get_subactions_for_actions(action_parsers): def omit_values_colliding_with_action_names(unparsed_arguments, parsed_arguments): ''' - Given a sequence of string arguments and a dict from action name to parsed argparse.Namespace - arguments, return the string arguments with any values omitted that happen to be the same as - the name of a borgmatic action. + Given unparsed arguments as a sequence of strings and a dict from action name to parsed + argparse.Namespace arguments, return the string arguments with any values omitted that happen to + be the same as the name of a borgmatic action. This prevents, for instance, "check --only extract" from triggering the "extract" action. ''' @@ -320,15 +320,15 @@ def make_argument_description(schema, flag_name): def add_array_element_arguments_from_schema(arguments_group, schema, unparsed_arguments, flag_name): - ''' + r''' Given an argparse._ArgumentGroup instance, a configuration schema dict, a sequence of unparsed argument strings, and a dotted flag name, convert the schema into corresponding command-line array element flags that correspond to the given unparsed arguments. Here's the background. We want to support flags that can have arbitrary indices like: - + --foo.bar[1].baz - + But argparse doesn't support that natively because the index can be an arbitrary number. We won't let that stop us though, will we? @@ -336,22 +336,22 @@ def add_array_element_arguments_from_schema(arguments_group, schema, unparsed_ar pattern that would match the flag name regardless of the number that's in it. The idea is that we want to look for unparsed arguments that appear like the flag name, but instead of "[0]" they have, say, "[1]" or "[123]". - + Next, we check each unparsed argument against that pattern. If one of them matches, add an argument flag for it to the argument parser group. Example: - + Let's say flag_name is: - + --foo.bar[0].baz - + ... then the regular expression pattern will be: - + ^--foo\.bar\[\d+\]\.baz - + ... and, if that matches an unparsed argument of: - + --foo.bar[1].baz - + ... then an argument flag will get added equal to that unparsed argument. And the unparsed argument will match it when parsing is performed! In this manner, we're using the actual user CLI input to inform what exact flags we support! @@ -359,7 +359,7 @@ def add_array_element_arguments_from_schema(arguments_group, schema, unparsed_ar if '[0]' not in flag_name or '--help' in unparsed_arguments: return - pattern = re.compile(f'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.")}$') + pattern = re.compile(fr'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.")}$') existing_flags = set( itertools.chain( *(group_action.option_strings for group_action in arguments_group._group_actions) @@ -409,10 +409,15 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names And if names are also passed in, they are considered to be the name components of an option (e.g. "foo" and "bar") and are used to construct a resulting flag. + + Bail if the schema is not a dict. ''' if names is None: names = () + if not isinstance(schema, dict): + return + schema_type = schema.get('type') # If this option has multiple types, just use the first one (that isn't "null"). @@ -449,13 +454,13 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names ) flag_name = '.'.join(names) - metavar = names[-1].upper() # Certain options already have corresponding flags on individual actions (like "create # --progress"), so don't bother adding them to the global flags. - if flag_name in OMITTED_FLAG_NAMES: + if not flag_name or flag_name in OMITTED_FLAG_NAMES: return + metavar = names[-1].upper() description = make_argument_description(schema, flag_name) argument_type = borgmatic.config.schema.parse_type(schema_type) full_flag_name = f"--{flag_name.replace('_', '-')}" @@ -482,10 +487,11 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names def make_parsers(schema, unparsed_arguments): ''' - Given a configuration schema dict, build a global arguments parser, individual action parsers, - and a combined parser containing both. Return them as a tuple. The global parser is useful for - parsing just global arguments while ignoring actions, and the combined parser is handy for - displaying help that includes everything: global flags, a list of actions, etc. + Given a configuration schema dict and unparsed arguments as a sequence of strings, build a + global arguments parser, individual action parsers, and a combined parser containing both. + Return them as a tuple. The global parser is useful for parsing just global arguments while + ignoring actions, and the combined parser is handy for displaying help that includes everything: + global flags, a list of actions, etc. ''' config_paths = collect.get_default_config_paths(expand_home=True) unexpanded_config_paths = collect.get_default_config_paths(expand_home=False) @@ -1752,8 +1758,8 @@ def make_parsers(schema, unparsed_arguments): def parse_arguments(schema, *unparsed_arguments): ''' Given a configuration schema dict and the command-line arguments with which this script was - invoked, parse the arguments and return them as a dict mapping from action name (or "global") to - an argparse.Namespace instance. + invoked and unparsed arguments as a sequence of strings, parse the arguments and return them as + a dict mapping from action name (or "global") to an argparse.Namespace instance. Raise ValueError if the arguments cannot be parsed. Raise SystemExit with an error code of 0 if "--help" was requested. diff --git a/borgmatic/commands/completion/bash.py b/borgmatic/commands/completion/bash.py index 6eb5e92d..c72fbbec 100644 --- a/borgmatic/commands/completion/bash.py +++ b/borgmatic/commands/completion/bash.py @@ -1,6 +1,7 @@ import borgmatic.commands.arguments import borgmatic.commands.completion.actions import borgmatic.commands.completion.flag +import borgmatic.config.validate def parser_flags(parser): @@ -10,7 +11,8 @@ def parser_flags(parser): ''' return ' '.join( flag_variant - for action in parser._actions for flag_name in action.option_strings + for action in parser._actions + for flag_name in action.option_strings for flag_variant in borgmatic.commands.completion.flag.variants(flag_name) ) diff --git a/borgmatic/commands/completion/flag.py b/borgmatic/commands/completion/flag.py index 67cf0712..6de6517f 100644 --- a/borgmatic/commands/completion/flag.py +++ b/borgmatic/commands/completion/flag.py @@ -11,6 +11,3 @@ def variants(flag_name): return yield flag_name - - - diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index 58360ad0..5ddc26c0 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -39,7 +39,9 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i if borgmatic.config.schema.compare_types(schema_type, {'array'}): config = ruamel.yaml.comments.CommentedSeq( example - if borgmatic.config.schema.compare_types(schema['items'].get('type'), SCALAR_SCHEMA_TYPES) + if borgmatic.config.schema.compare_types( + schema['items'].get('type'), SCALAR_SCHEMA_TYPES + ) else [ schema_to_sample_configuration( schema['items'], source_config, level, parent_is_sequence=True diff --git a/borgmatic/config/schema.py b/borgmatic/config/schema.py index d4698c3a..0ce774d9 100644 --- a/borgmatic/config/schema.py +++ b/borgmatic/config/schema.py @@ -50,9 +50,7 @@ def compare_types(schema_type, target_types, match=any): compare elements. ''' if isinstance(schema_type, list): - if match( - element_schema_type in target_types for element_schema_type in schema_type - ): + if match(element_schema_type in target_types for element_schema_type in schema_type): return True return False diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 5899de7d..29215f91 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -21,6 +21,18 @@ def schema_filename(): return schema_path +def load_schema(schema_path): + ''' + Given a schema filename path, load the schema and return it as a dict. + + Raise Validation_error if the schema could not be parsed. + ''' + try: + return load.load_configuration(schema_path) + except (ruamel.yaml.error.YAMLError, RecursionError) as error: + raise Validation_error(schema_path, (str(error),)) + + def format_json_error_path_element(path_element): ''' Given a path element into a JSON data structure, format it for display as a string. diff --git a/tests/integration/borg/test_commands.py b/tests/integration/borg/test_commands.py index 4e9261e8..80de7dcd 100644 --- a/tests/integration/borg/test_commands.py +++ b/tests/integration/borg/test_commands.py @@ -53,7 +53,7 @@ def fuzz_argument(arguments, argument_name): def test_transfer_archives_command_does_not_duplicate_flags_or_raise(): arguments = borgmatic.commands.arguments.parse_arguments( - 'transfer', '--source-repository', 'foo' + {}, 'transfer', '--source-repository', 'foo' )['transfer'] flexmock(borgmatic.borg.transfer).should_receive('execute_command').replace_with( assert_command_does_not_duplicate_flags @@ -74,7 +74,7 @@ def test_transfer_archives_command_does_not_duplicate_flags_or_raise(): def test_prune_archives_command_does_not_duplicate_flags_or_raise(): - arguments = borgmatic.commands.arguments.parse_arguments('prune')['prune'] + arguments = borgmatic.commands.arguments.parse_arguments({}, 'prune')['prune'] flexmock(borgmatic.borg.prune).should_receive('execute_command').replace_with( assert_command_does_not_duplicate_flags ) @@ -94,7 +94,7 @@ def test_prune_archives_command_does_not_duplicate_flags_or_raise(): def test_mount_archive_command_does_not_duplicate_flags_or_raise(): - arguments = borgmatic.commands.arguments.parse_arguments('mount', '--mount-point', 'tmp')[ + arguments = borgmatic.commands.arguments.parse_arguments({}, 'mount', '--mount-point', 'tmp')[ 'mount' ] flexmock(borgmatic.borg.mount).should_receive('execute_command').replace_with( @@ -116,7 +116,7 @@ def test_mount_archive_command_does_not_duplicate_flags_or_raise(): def test_make_list_command_does_not_duplicate_flags_or_raise(): - arguments = borgmatic.commands.arguments.parse_arguments('list')['list'] + arguments = borgmatic.commands.arguments.parse_arguments({}, 'list')['list'] for argument_name in dir(arguments): if argument_name.startswith('_'): @@ -134,7 +134,7 @@ def test_make_list_command_does_not_duplicate_flags_or_raise(): def test_make_repo_list_command_does_not_duplicate_flags_or_raise(): - arguments = borgmatic.commands.arguments.parse_arguments('repo-list')['repo-list'] + arguments = borgmatic.commands.arguments.parse_arguments({}, 'repo-list')['repo-list'] for argument_name in dir(arguments): if argument_name.startswith('_'): @@ -152,7 +152,7 @@ def test_make_repo_list_command_does_not_duplicate_flags_or_raise(): def test_display_archives_info_command_does_not_duplicate_flags_or_raise(): - arguments = borgmatic.commands.arguments.parse_arguments('info')['info'] + arguments = borgmatic.commands.arguments.parse_arguments({}, 'info')['info'] flexmock(borgmatic.borg.info).should_receive('execute_command_and_capture_output').replace_with( assert_command_does_not_duplicate_flags ) diff --git a/tests/integration/commands/completion/test_actions.py b/tests/integration/commands/completion/test_actions.py index 2e6fde9b..52dfd268 100644 --- a/tests/integration/commands/completion/test_actions.py +++ b/tests/integration/commands/completion/test_actions.py @@ -1,4 +1,5 @@ import borgmatic.commands.arguments +import borgmatic.config.validate from borgmatic.commands.completion import actions as module @@ -7,7 +8,10 @@ def test_available_actions_uses_only_subactions_for_action_with_subactions(): unused_global_parser, action_parsers, unused_combined_parser, - ) = borgmatic.commands.arguments.make_parsers() + ) = borgmatic.commands.arguments.make_parsers( + schema=borgmatic.config.validate.load_schema(borgmatic.config.validate.schema_filename()), + unparsed_arguments=(), + ) actions = module.available_actions(action_parsers, 'config') @@ -20,7 +24,10 @@ def test_available_actions_omits_subactions_for_action_without_subactions(): unused_global_parser, action_parsers, unused_combined_parser, - ) = borgmatic.commands.arguments.make_parsers() + ) = borgmatic.commands.arguments.make_parsers( + schema=borgmatic.config.validate.load_schema(borgmatic.config.validate.schema_filename()), + unparsed_arguments=(), + ) actions = module.available_actions(action_parsers, 'list') diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 4399c867..3469858c 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -8,7 +8,7 @@ def test_parse_arguments_with_no_arguments_uses_defaults(): config_paths = ['default'] flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) - arguments = module.parse_arguments() + arguments = module.parse_arguments({}) global_arguments = arguments['global'] assert global_arguments.config_paths == config_paths @@ -21,7 +21,7 @@ def test_parse_arguments_with_no_arguments_uses_defaults(): def test_parse_arguments_with_multiple_config_flags_parses_as_list(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--config', 'myconfig', '--config', 'otherconfig') + arguments = module.parse_arguments({}, '--config', 'myconfig', '--config', 'otherconfig') global_arguments = arguments['global'] assert global_arguments.config_paths == ['myconfig', 'otherconfig'] @@ -34,7 +34,7 @@ def test_parse_arguments_with_multiple_config_flags_parses_as_list(): def test_parse_arguments_with_action_after_config_path_omits_action(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--config', 'myconfig', 'list', '--json') + arguments = module.parse_arguments({}, '--config', 'myconfig', 'list', '--json') global_arguments = arguments['global'] assert global_arguments.config_paths == ['myconfig'] @@ -45,7 +45,9 @@ def test_parse_arguments_with_action_after_config_path_omits_action(): def test_parse_arguments_with_action_after_config_path_omits_aliased_action(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey') + arguments = module.parse_arguments( + {}, '--config', 'myconfig', 'init', '--encryption', 'repokey' + ) global_arguments = arguments['global'] assert global_arguments.config_paths == ['myconfig'] @@ -56,7 +58,7 @@ def test_parse_arguments_with_action_after_config_path_omits_aliased_action(): def test_parse_arguments_with_action_and_positional_arguments_after_config_path_omits_action_and_arguments(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--config', 'myconfig', 'borg', 'key', 'export') + arguments = module.parse_arguments({}, '--config', 'myconfig', 'borg', 'key', 'export') global_arguments = arguments['global'] assert global_arguments.config_paths == ['myconfig'] @@ -68,7 +70,7 @@ def test_parse_arguments_with_verbosity_overrides_default(): config_paths = ['default'] flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) - arguments = module.parse_arguments('--verbosity', '1') + arguments = module.parse_arguments({}, '--verbosity', '1') global_arguments = arguments['global'] assert global_arguments.config_paths == config_paths @@ -82,7 +84,7 @@ def test_parse_arguments_with_syslog_verbosity_overrides_default(): config_paths = ['default'] flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) - arguments = module.parse_arguments('--syslog-verbosity', '2') + arguments = module.parse_arguments({}, '--syslog-verbosity', '2') global_arguments = arguments['global'] assert global_arguments.config_paths == config_paths @@ -96,7 +98,7 @@ def test_parse_arguments_with_log_file_verbosity_overrides_default(): config_paths = ['default'] flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) - arguments = module.parse_arguments('--log-file-verbosity', '-1') + arguments = module.parse_arguments({}, '--log-file-verbosity', '-1') global_arguments = arguments['global'] assert global_arguments.config_paths == config_paths @@ -109,7 +111,7 @@ def test_parse_arguments_with_log_file_verbosity_overrides_default(): def test_parse_arguments_with_single_override_parses(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--override', 'foo.bar=baz') + arguments = module.parse_arguments({}, '--override', 'foo.bar=baz') global_arguments = arguments['global'] assert global_arguments.overrides == ['foo.bar=baz'] @@ -119,7 +121,7 @@ def test_parse_arguments_with_multiple_overrides_flags_parses(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) arguments = module.parse_arguments( - '--override', 'foo.bar=baz', '--override', 'foo.quux=7', '--override', 'this.that=8' + {}, '--override', 'foo.bar=baz', '--override', 'foo.quux=7', '--override', 'this.that=8' ) global_arguments = arguments['global'] @@ -127,7 +129,7 @@ def test_parse_arguments_with_multiple_overrides_flags_parses(): def test_parse_arguments_with_list_json_overrides_default(): - arguments = module.parse_arguments('list', '--json') + arguments = module.parse_arguments({}, 'list', '--json') assert 'list' in arguments assert arguments['list'].json is True @@ -136,7 +138,7 @@ def test_parse_arguments_with_list_json_overrides_default(): def test_parse_arguments_with_no_actions_defaults_to_all_actions_enabled(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments() + arguments = module.parse_arguments({}) assert 'prune' in arguments assert 'create' in arguments @@ -146,7 +148,7 @@ def test_parse_arguments_with_no_actions_defaults_to_all_actions_enabled(): def test_parse_arguments_with_no_actions_passes_argument_to_relevant_actions(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--stats', '--list') + arguments = module.parse_arguments({}, '--stats', '--list') assert 'prune' in arguments assert arguments['prune'].stats @@ -161,7 +163,7 @@ def test_parse_arguments_with_help_and_no_actions_shows_global_help(capsys): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit) as exit: - module.parse_arguments('--help') + module.parse_arguments({}, '--help') assert exit.value.code == 0 captured = capsys.readouterr() @@ -173,7 +175,7 @@ def test_parse_arguments_with_help_and_action_shows_action_help(capsys): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit) as exit: - module.parse_arguments('create', '--help') + module.parse_arguments({}, 'create', '--help') assert exit.value.code == 0 captured = capsys.readouterr() @@ -185,7 +187,7 @@ def test_parse_arguments_with_help_and_action_shows_action_help(capsys): def test_parse_arguments_with_action_before_global_options_parses_options(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('prune', '--verbosity', '2') + arguments = module.parse_arguments({}, 'prune', '--verbosity', '2') assert 'prune' in arguments assert arguments['global'].verbosity == 2 @@ -194,7 +196,7 @@ def test_parse_arguments_with_action_before_global_options_parses_options(): def test_parse_arguments_with_global_options_before_action_parses_options(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--verbosity', '2', 'prune') + arguments = module.parse_arguments({}, '--verbosity', '2', 'prune') assert 'prune' in arguments assert arguments['global'].verbosity == 2 @@ -203,7 +205,7 @@ def test_parse_arguments_with_global_options_before_action_parses_options(): def test_parse_arguments_with_prune_action_leaves_other_actions_disabled(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('prune') + arguments = module.parse_arguments({}, 'prune') assert 'prune' in arguments assert 'create' not in arguments @@ -213,7 +215,7 @@ def test_parse_arguments_with_prune_action_leaves_other_actions_disabled(): def test_parse_arguments_with_multiple_actions_leaves_other_action_disabled(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('create', 'check') + arguments = module.parse_arguments({}, 'create', 'check') assert 'prune' not in arguments assert 'create' in arguments @@ -224,60 +226,53 @@ def test_parse_arguments_disallows_invalid_argument(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--posix-me-harder') + module.parse_arguments({}, '--posix-me-harder') def test_parse_arguments_disallows_encryption_mode_without_init(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--config', 'myconfig', '--encryption', 'repokey') + module.parse_arguments({}, '--config', 'myconfig', '--encryption', 'repokey') def test_parse_arguments_allows_encryption_mode_with_init(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey') - - -def test_parse_arguments_requires_encryption_mode_with_init(): - flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - - with pytest.raises(SystemExit): - module.parse_arguments('--config', 'myconfig', 'init') + module.parse_arguments({}, '--config', 'myconfig', 'init', '--encryption', 'repokey') def test_parse_arguments_disallows_append_only_without_init(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--config', 'myconfig', '--append-only') + module.parse_arguments({}, '--config', 'myconfig', '--append-only') def test_parse_arguments_disallows_storage_quota_without_init(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--config', 'myconfig', '--storage-quota', '5G') + module.parse_arguments({}, '--config', 'myconfig', '--storage-quota', '5G') def test_parse_arguments_allows_init_and_prune(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey', 'prune') + module.parse_arguments({}, '--config', 'myconfig', 'init', '--encryption', 'repokey', 'prune') def test_parse_arguments_allows_init_and_create(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey', 'create') + module.parse_arguments({}, '--config', 'myconfig', 'init', '--encryption', 'repokey', 'create') def test_parse_arguments_allows_repository_with_extract(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) module.parse_arguments( - '--config', 'myconfig', 'extract', '--repository', 'test.borg', '--archive', 'test' + {}, '--config', 'myconfig', 'extract', '--repository', 'test.borg', '--archive', 'test' ) @@ -285,6 +280,7 @@ def test_parse_arguments_allows_repository_with_mount(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) module.parse_arguments( + {}, '--config', 'myconfig', 'mount', @@ -300,187 +296,187 @@ def test_parse_arguments_allows_repository_with_mount(): def test_parse_arguments_allows_repository_with_list(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'list', '--repository', 'test.borg') + module.parse_arguments({}, '--config', 'myconfig', 'list', '--repository', 'test.borg') def test_parse_arguments_disallows_archive_unless_action_consumes_it(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--config', 'myconfig', '--archive', 'test') + module.parse_arguments({}, '--config', 'myconfig', '--archive', 'test') def test_parse_arguments_disallows_paths_unless_action_consumes_it(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--config', 'myconfig', '--path', 'test') + module.parse_arguments({}, '--config', 'myconfig', '--path', 'test') def test_parse_arguments_disallows_other_actions_with_config_bootstrap(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('config', 'bootstrap', '--repository', 'test.borg', 'list') + module.parse_arguments({}, 'config', 'bootstrap', '--repository', 'test.borg', 'list') def test_parse_arguments_allows_archive_with_extract(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'extract', '--archive', 'test') + module.parse_arguments({}, '--config', 'myconfig', 'extract', '--archive', 'test') def test_parse_arguments_allows_archive_with_mount(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) module.parse_arguments( - '--config', 'myconfig', 'mount', '--archive', 'test', '--mount-point', '/mnt' + {}, '--config', 'myconfig', 'mount', '--archive', 'test', '--mount-point', '/mnt' ) def test_parse_arguments_allows_archive_with_restore(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'restore', '--archive', 'test') + module.parse_arguments({}, '--config', 'myconfig', 'restore', '--archive', 'test') def test_parse_arguments_allows_archive_with_list(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'list', '--archive', 'test') + module.parse_arguments({}, '--config', 'myconfig', 'list', '--archive', 'test') def test_parse_arguments_requires_archive_with_extract(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit): - module.parse_arguments('--config', 'myconfig', 'extract') + module.parse_arguments({}, '--config', 'myconfig', 'extract') def test_parse_arguments_requires_archive_with_restore(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit): - module.parse_arguments('--config', 'myconfig', 'restore') + module.parse_arguments({}, '--config', 'myconfig', 'restore') def test_parse_arguments_requires_mount_point_with_mount(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit): - module.parse_arguments('--config', 'myconfig', 'mount', '--archive', 'test') + module.parse_arguments({}, '--config', 'myconfig', 'mount', '--archive', 'test') def test_parse_arguments_requires_mount_point_with_umount(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit): - module.parse_arguments('--config', 'myconfig', 'umount') + module.parse_arguments({}, '--config', 'myconfig', 'umount') def test_parse_arguments_allows_progress_before_create(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--progress', 'create', 'list') + module.parse_arguments({}, '--progress', 'create', 'list') def test_parse_arguments_allows_progress_after_create(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('create', '--progress', 'list') + module.parse_arguments({}, 'create', '--progress', 'list') def test_parse_arguments_allows_progress_and_extract(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--progress', 'extract', '--archive', 'test', 'list') + module.parse_arguments({}, '--progress', 'extract', '--archive', 'test', 'list') def test_parse_arguments_disallows_progress_without_create(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--progress', 'list') + module.parse_arguments({}, '--progress', 'list') def test_parse_arguments_with_stats_and_create_flags_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--stats', 'create', 'list') + module.parse_arguments({}, '--stats', 'create', 'list') def test_parse_arguments_with_stats_and_prune_flags_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--stats', 'prune', 'list') + module.parse_arguments({}, '--stats', 'prune', 'list') def test_parse_arguments_with_stats_flag_but_no_create_or_prune_flag_raises_value_error(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--stats', 'list') + module.parse_arguments({}, '--stats', 'list') def test_parse_arguments_with_list_and_create_flags_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--list', 'create') + module.parse_arguments({}, '--list', 'create') def test_parse_arguments_with_list_and_prune_flags_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--list', 'prune') + module.parse_arguments({}, '--list', 'prune') def test_parse_arguments_with_list_flag_but_no_relevant_action_raises_value_error(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - with pytest.raises(SystemExit): - module.parse_arguments('--list', 'repo-create') + with pytest.raises(ValueError): + module.parse_arguments({}, '--list', 'repo-create') def test_parse_arguments_disallows_list_with_progress_for_create_action(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('create', '--list', '--progress') + module.parse_arguments({}, 'create', '--list', '--progress') def test_parse_arguments_disallows_list_with_json_for_create_action(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('create', '--list', '--json') + module.parse_arguments({}, 'create', '--list', '--json') def test_parse_arguments_allows_json_with_list_or_info(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('list', '--json') - module.parse_arguments('info', '--json') + module.parse_arguments({}, 'list', '--json') + module.parse_arguments({}, 'info', '--json') def test_parse_arguments_disallows_json_with_both_list_and_info(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('list', 'info', '--json') + module.parse_arguments({}, 'list', 'info', '--json') def test_parse_arguments_disallows_json_with_both_list_and_repo_info(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('list', 'repo-info', '--json') + module.parse_arguments({}, 'list', 'repo-info', '--json') def test_parse_arguments_disallows_json_with_both_repo_info_and_info(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('repo-info', 'info', '--json') + module.parse_arguments({}, 'repo-info', 'info', '--json') def test_parse_arguments_disallows_transfer_with_both_archive_and_match_archives(): @@ -488,6 +484,7 @@ def test_parse_arguments_disallows_transfer_with_both_archive_and_match_archives with pytest.raises(ValueError): module.parse_arguments( + {}, 'transfer', '--source-repository', 'source.borg', @@ -502,74 +499,74 @@ def test_parse_arguments_disallows_list_with_both_prefix_and_match_archives(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('list', '--prefix', 'foo', '--match-archives', 'sh:*bar') + module.parse_arguments({}, 'list', '--prefix', 'foo', '--match-archives', 'sh:*bar') def test_parse_arguments_disallows_repo_list_with_both_prefix_and_match_archives(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('repo-list', '--prefix', 'foo', '--match-archives', 'sh:*bar') + module.parse_arguments({}, 'repo-list', '--prefix', 'foo', '--match-archives', 'sh:*bar') def test_parse_arguments_disallows_info_with_both_archive_and_match_archives(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('info', '--archive', 'foo', '--match-archives', 'sh:*bar') + module.parse_arguments({}, 'info', '--archive', 'foo', '--match-archives', 'sh:*bar') def test_parse_arguments_disallows_info_with_both_archive_and_prefix(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('info', '--archive', 'foo', '--prefix', 'bar') + module.parse_arguments({}, 'info', '--archive', 'foo', '--prefix', 'bar') def test_parse_arguments_disallows_info_with_both_prefix_and_match_archives(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('info', '--prefix', 'foo', '--match-archives', 'sh:*bar') + module.parse_arguments({}, 'info', '--prefix', 'foo', '--match-archives', 'sh:*bar') def test_parse_arguments_check_only_extract_does_not_raise_extract_subparser_error(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('check', '--only', 'extract') + module.parse_arguments({}, 'check', '--only', 'extract') def test_parse_arguments_extract_archive_check_does_not_raise_check_subparser_error(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('extract', '--archive', 'check') + module.parse_arguments({}, 'extract', '--archive', 'check') def test_parse_arguments_extract_with_check_only_extract_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('extract', '--archive', 'name', 'check', '--only', 'extract') + module.parse_arguments({}, 'extract', '--archive', 'name', 'check', '--only', 'extract') def test_parse_arguments_bootstrap_without_config_errors(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('bootstrap') + module.parse_arguments({}, 'bootstrap') def test_parse_arguments_config_with_no_subaction_errors(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('config') + module.parse_arguments({}, 'config') def test_parse_arguments_config_with_help_shows_config_help(capsys): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit) as exit: - module.parse_arguments('config', '--help') + module.parse_arguments({}, 'config', '--help') assert exit.value.code == 0 captured = capsys.readouterr() @@ -582,7 +579,7 @@ def test_parse_arguments_config_with_subaction_but_missing_flags_errors(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit) as exit: - module.parse_arguments('config', 'bootstrap') + module.parse_arguments({}, 'config', 'bootstrap') assert exit.value.code == 2 @@ -591,7 +588,7 @@ def test_parse_arguments_config_with_subaction_and_help_shows_subaction_help(cap flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit) as exit: - module.parse_arguments('config', 'bootstrap', '--help') + module.parse_arguments({}, 'config', 'bootstrap', '--help') assert exit.value.code == 0 captured = capsys.readouterr() @@ -601,26 +598,30 @@ def test_parse_arguments_config_with_subaction_and_help_shows_subaction_help(cap def test_parse_arguments_config_with_subaction_and_required_flags_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('config', 'bootstrap', '--repository', 'repo.borg') + module.parse_arguments({}, 'config', 'bootstrap', '--repository', 'repo.borg') def test_parse_arguments_config_with_subaction_and_global_flags_at_start_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--verbosity', '1', 'config', 'bootstrap', '--repository', 'repo.borg') + module.parse_arguments( + {}, '--verbosity', '1', 'config', 'bootstrap', '--repository', 'repo.borg' + ) def test_parse_arguments_config_with_subaction_and_global_flags_at_end_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('config', 'bootstrap', '--repository', 'repo.borg', '--verbosity', '1') + module.parse_arguments( + {}, 'config', 'bootstrap', '--repository', 'repo.borg', '--verbosity', '1' + ) def test_parse_arguments_config_with_subaction_and_explicit_config_file_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) module.parse_arguments( - 'config', 'bootstrap', '--repository', 'repo.borg', '--config', 'test.yaml' + {}, 'config', 'bootstrap', '--repository', 'repo.borg', '--config', 'test.yaml' ) @@ -628,10 +629,23 @@ def test_parse_arguments_with_borg_action_and_dry_run_raises(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--dry-run', 'borg', 'list') + module.parse_arguments({}, '--dry-run', 'borg', 'list') def test_parse_arguments_with_borg_action_and_no_dry_run_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('borg', 'list') + module.parse_arguments({}, 'borg', 'list') + + +def test_parse_arguments_with_argument_from_schema_does_not_raise(): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + module.parse_arguments( + { + 'type': 'object', + 'properties': {'foo': {'type': 'object', 'properties': {'bar': {'type': 'integer'}}}}, + }, + '--foo.bar', + '3', + ) diff --git a/tests/integration/config/test_generate.py b/tests/integration/config/test_generate.py index 42622403..71c0abc5 100644 --- a/tests/integration/config/test_generate.py +++ b/tests/integration/config/test_generate.py @@ -21,9 +21,9 @@ def test_schema_to_sample_configuration_comments_out_non_default_options(): 'type': 'object', 'properties': dict( [ - ('field1', {'example': 'Example 1'}), - ('field2', {'example': 'Example 2'}), - ('source_directories', {'example': 'Example 3'}), + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ('source_directories', {'type': 'string', 'example': 'Example 3'}), ] ), } @@ -47,9 +47,9 @@ def test_schema_to_sample_configuration_comments_out_non_source_config_options() 'type': 'object', 'properties': dict( [ - ('field1', {'example': 'Example 1'}), - ('field2', {'example': 'Example 2'}), - ('field3', {'example': 'Example 3'}), + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ('field3', {'type': 'string', 'example': 'Example 3'}), ] ), } @@ -76,9 +76,9 @@ def test_schema_to_sample_configuration_comments_out_non_default_options_in_sequ 'type': 'object', 'properties': dict( [ - ('field1', {'example': 'Example 1'}), - ('field2', {'example': 'Example 2'}), - ('source_directories', {'example': 'Example 3'}), + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ('source_directories', {'type': 'string', 'example': 'Example 3'}), ] ), }, @@ -105,9 +105,9 @@ def test_schema_to_sample_configuration_comments_out_non_source_config_options_i 'type': 'object', 'properties': dict( [ - ('field1', {'example': 'Example 1'}), - ('field2', {'example': 'Example 2'}), - ('field3', {'example': 'Example 3'}), + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ('field3', {'type': 'string', 'example': 'Example 3'}), ] ), }, diff --git a/tests/integration/config/test_validate.py b/tests/integration/config/test_validate.py index 9cd5c980..61d12d7f 100644 --- a/tests/integration/config/test_validate.py +++ b/tests/integration/config/test_validate.py @@ -58,7 +58,9 @@ def test_parse_configuration_transforms_file_into_mapping(): ''' ) - config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + config, config_paths, logs = module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + ) assert config == { 'source_directories': ['/home', '/etc'], @@ -86,7 +88,9 @@ def test_parse_configuration_passes_through_quoted_punctuation(): ''' ) - config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + config, config_paths, logs = module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + ) assert config == { 'source_directories': [f'/home/{string.punctuation}'], @@ -119,7 +123,7 @@ def test_parse_configuration_with_schema_lacking_examples_does_not_raise(): ''', ) - module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock()) def test_parse_configuration_inlines_include_inside_deprecated_section(): @@ -145,7 +149,9 @@ def test_parse_configuration_inlines_include_inside_deprecated_section(): include_file.name = 'include.yaml' builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) - config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + config, config_paths, logs = module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + ) assert config == { 'source_directories': ['/home'], @@ -181,7 +187,9 @@ def test_parse_configuration_merges_include(): include_file.name = 'include.yaml' builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) - config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + config, config_paths, logs = module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + ) assert config == { 'source_directories': ['/home'], @@ -196,7 +204,9 @@ def test_parse_configuration_merges_include(): def test_parse_configuration_raises_for_missing_config_file(): with pytest.raises(FileNotFoundError): - module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + ) def test_parse_configuration_raises_for_missing_schema_file(): @@ -208,14 +218,18 @@ def test_parse_configuration_raises_for_missing_schema_file(): builtins.should_receive('open').with_args('/tmp/schema.yaml').and_raise(FileNotFoundError) with pytest.raises(FileNotFoundError): - module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + ) def test_parse_configuration_raises_for_syntax_error(): mock_config_and_schema('foo:\nbar') with pytest.raises(ValueError): - module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + ) def test_parse_configuration_raises_for_validation_error(): @@ -228,7 +242,9 @@ def test_parse_configuration_raises_for_validation_error(): ) with pytest.raises(module.Validation_error): - module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + ) def test_parse_configuration_applies_overrides(): @@ -245,7 +261,10 @@ def test_parse_configuration_applies_overrides(): ) config, config_paths, logs = module.parse_configuration( - '/tmp/config.yaml', '/tmp/schema.yaml', overrides=['local_path=borg2'] + '/tmp/config.yaml', + '/tmp/schema.yaml', + global_arguments=flexmock(), + overrides=['local_path=borg2'], ) assert config == { @@ -273,7 +292,9 @@ def test_parse_configuration_applies_normalization_after_environment_variable_in ) flexmock(os).should_receive('getenv').replace_with(lambda variable_name, default: default) - config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + config, config_paths, logs = module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + ) assert config == { 'source_directories': ['/home'], diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 160f4454..9faa6632 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -1537,6 +1537,7 @@ def test_load_configurations_collects_parsed_configurations_and_logs(resolve_env configs, config_paths, logs = tuple( module.load_configurations( ('test.yaml', 'other.yaml'), + global_arguments=flexmock(), resolve_env=resolve_env, ) ) @@ -1549,7 +1550,9 @@ def test_load_configurations_collects_parsed_configurations_and_logs(resolve_env def test_load_configurations_logs_warning_for_permission_error(): flexmock(module.validate).should_receive('parse_configuration').and_raise(PermissionError) - configs, config_paths, logs = tuple(module.load_configurations(('test.yaml',))) + configs, config_paths, logs = tuple( + module.load_configurations(('test.yaml',), global_arguments=flexmock()) + ) assert configs == {} assert config_paths == [] @@ -1559,7 +1562,9 @@ def test_load_configurations_logs_warning_for_permission_error(): def test_load_configurations_logs_critical_for_parse_error(): flexmock(module.validate).should_receive('parse_configuration').and_raise(ValueError) - configs, config_paths, logs = tuple(module.load_configurations(('test.yaml',))) + configs, config_paths, logs = tuple( + module.load_configurations(('test.yaml',), global_arguments=flexmock()) + ) assert configs == {} assert config_paths == [] diff --git a/tests/unit/config/test_generate.py b/tests/unit/config/test_generate.py index 9c07001d..885bdcca 100644 --- a/tests/unit/config/test_generate.py +++ b/tests/unit/config/test_generate.py @@ -17,9 +17,15 @@ def test_schema_to_sample_configuration_generates_config_map_with_examples(): ), } flexmock(module.borgmatic.config.schema).should_receive('compare_types').and_return(False) - flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('object', {'object'}).and_return(True) - flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('string', module.SCALAR_SCHEMA_TYPES, match=all).and_return(True) - flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return(schema['properties']) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'object', {'object'} + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'string', module.SCALAR_SCHEMA_TYPES, match=all + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return( + schema['properties'] + ) flexmock(module.ruamel.yaml.comments).should_receive('CommentedMap').replace_with(dict) flexmock(module).should_receive('add_comments_to_configuration_object') @@ -50,15 +56,26 @@ def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_e 'items': { 'type': 'object', 'properties': dict( - [('field1', {'type': 'string', 'example': 'Example 1'}), ('field2', {'type': 'string', 'example': 'Example 2'})] + [ + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ] ), }, } flexmock(module.borgmatic.config.schema).should_receive('compare_types').and_return(False) - flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('array', {'array'}).and_return(True) - flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('object', {'object'}).and_return(True) - flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('string', module.SCALAR_SCHEMA_TYPES, match=all).and_return(True) - flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return(schema['items']['properties']) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'array', {'array'} + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'object', {'object'} + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'string', module.SCALAR_SCHEMA_TYPES, match=all + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return( + schema['items']['properties'] + ) flexmock(module.ruamel.yaml.comments).should_receive('CommentedSeq').replace_with(list) flexmock(module).should_receive('add_comments_to_configuration_sequence') flexmock(module).should_receive('add_comments_to_configuration_object') @@ -74,15 +91,26 @@ def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_m 'items': { 'type': ['object', 'null'], 'properties': dict( - [('field1', {'type': 'string', 'example': 'Example 1'}), ('field2', {'type': 'string', 'example': 'Example 2'})] + [ + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ] ), }, } flexmock(module.borgmatic.config.schema).should_receive('compare_types').and_return(False) - flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('array', {'array'}).and_return(True) - flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args(['object', 'null'], {'object'}).and_return(True) - flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args('string', module.SCALAR_SCHEMA_TYPES, match=all).and_return(True) - flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return(schema['items']['properties']) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'array', {'array'} + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + ['object', 'null'], {'object'} + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'string', module.SCALAR_SCHEMA_TYPES, match=all + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return( + schema['items']['properties'] + ) flexmock(module.ruamel.yaml.comments).should_receive('CommentedSeq').replace_with(list) flexmock(module).should_receive('add_comments_to_configuration_sequence') flexmock(module).should_receive('add_comments_to_configuration_object') From 16a1121649042ea9735dca5bfb0d8f21d5f634d5 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 23 Mar 2025 18:45:49 -0700 Subject: [PATCH 152/226] Get existing end-to-end tests passing (#303). --- borgmatic/config/schema.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index a7c9fc4c..078c7237 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -37,7 +37,7 @@ properties: path: type: string description: The local path or Borg URL of the repository. - example: ssh://user@backupserver/./{fqdn} + example: ssh://user@backupserver/./sourcehostname.borg label: type: string description: | From 65b1d8e8b22807af005d4c94e6dac35fad8d4ede Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 23 Mar 2025 19:13:07 -0700 Subject: [PATCH 153/226] Clarify NEWS items (#303). --- NEWS | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index 61b95f34..8de5ff53 100644 --- a/NEWS +++ b/NEWS @@ -1,11 +1,11 @@ 2.0.0.dev0 - * #303: Add flags for setting any borgmatic configuration option from the command-line. See the + * #303: Add command-line flags for every borgmatic configuration option. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides - * #303: Add configuration options that serve as defaults for some (but not all) borgmatic + * #303: Add configuration options that serve as defaults for some (but not all) command-line action flags. For example, each entry in "repositories:" now has an "encryption" option that - applies to the "repo-create" action. See the documentation for more information: - https://torsion.org/borgmatic/docs/reference/configuration/ + applies to the "repo-create" action, serving as a default for the "--encryption" flag. See the + documentation for more information: https://torsion.org/borgmatic/docs/reference/configuration/ * #345: Add a "key import" action to import a repository key from backup. * #790, #821: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more flexible "commands:". See the documentation for more information: From d651813601e14239662cdb775fa1648b147a0f88 Mon Sep 17 00:00:00 2001 From: Gautam Aggarwal Date: Mon, 24 Mar 2025 03:39:26 +0000 Subject: [PATCH 154/226] Custom command options for MongoDB hook #837 --- borgmatic/config/schema.yaml | 18 +++ borgmatic/hooks/data_source/mongodb.py | 14 +- tests/unit/hooks/data_source/test_mongodb.py | 135 ++++++++++++++++++- 3 files changed, 157 insertions(+), 10 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 7b309ea0..ffbde053 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1726,6 +1726,24 @@ properties: dump command, without performing any validation on them. See mongorestore documentation for details. example: --restoreDbUsersAndRoles + mongodump_command: + type: string + description: | + Command to use instead of "mongodump". This can be used to + run a specific mongodump version (e.g., one inside a + running container). If you run it from within a + container, make sure to mount the path in the + "user_runtime_directory" option from the host into the + container at the same location. Defaults to "mongodump". + example: docker exec mongodb_container mongodump + mongorestore_command: + type: string + description: | + Command to run when restoring a database instead + of "mongorestore". This can be used to run a specific + mongorestore version (e.g., one inside a running container). + Defaults to "mongorestore". + example: docker exec mongodb_container mongorestore description: | List of one or more MongoDB databases to dump before creating a backup, run once per configuration file. The database dumps are diff --git a/borgmatic/hooks/data_source/mongodb.py b/borgmatic/hooks/data_source/mongodb.py index ff22d8c4..74ae15f3 100644 --- a/borgmatic/hooks/data_source/mongodb.py +++ b/borgmatic/hooks/data_source/mongodb.py @@ -114,14 +114,17 @@ def make_password_config_file(password): def build_dump_command(database, config, dump_filename, dump_format): ''' - Return the mongodump command from a single database configuration. + Return the custom mongodump_command from a single database configuration. ''' all_databases = database['name'] == 'all' password = borgmatic.hooks.credential.parse.resolve_credential(database.get('password'), config) + dump_command = tuple( + shlex.quote(part) for part in shlex.split(database.get('mongodump_command') or 'mongodump') + ) return ( - ('mongodump',) + dump_command + (('--out', shlex.quote(dump_filename)) if dump_format == 'directory' else ()) + (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ()) + (('--port', shlex.quote(str(database['port']))) if 'port' in database else ()) @@ -230,7 +233,7 @@ def restore_data_source_dump( def build_restore_command(extract_process, database, config, dump_filename, connection_params): ''' - Return the mongorestore command from a single database configuration. + Return the custom mongorestore_command from a single database configuration. ''' hostname = connection_params['hostname'] or database.get( 'restore_hostname', database.get('hostname') @@ -251,7 +254,10 @@ def build_restore_command(extract_process, database, config, dump_filename, conn config, ) - command = ['mongorestore'] + command = list( + shlex.quote(part) + for part in shlex.split(database.get('mongorestore_command') or 'mongorestore') + ) if extract_process: command.append('--archive') else: diff --git a/tests/unit/hooks/data_source/test_mongodb.py b/tests/unit/hooks/data_source/test_mongodb.py index 0cef73f8..bb8532c1 100644 --- a/tests/unit/hooks/data_source/test_mongodb.py +++ b/tests/unit/hooks/data_source/test_mongodb.py @@ -24,6 +24,9 @@ def test_use_streaming_false_for_no_databases(): def test_dump_data_sources_runs_mongodump_for_each_database(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'foo'}, {'name': 'bar'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') @@ -53,6 +56,9 @@ def test_dump_data_sources_runs_mongodump_for_each_database(): def test_dump_data_sources_with_dry_run_skips_mongodump(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'foo'}, {'name': 'bar'}] flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( @@ -75,6 +81,9 @@ def test_dump_data_sources_with_dry_run_skips_mongodump(): def test_dump_data_sources_runs_mongodump_with_hostname_and_port(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') @@ -111,9 +120,12 @@ def test_dump_data_sources_runs_mongodump_with_hostname_and_port(): def test_dump_data_sources_runs_mongodump_with_username_and_password(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [ { - 'name': 'foo', + 'name': 'foo', # Ensure this matches the expected format in the related functions 'username': 'mongo', 'password': 'trustsome1', 'authentication_database': 'admin', @@ -162,6 +174,9 @@ def test_dump_data_sources_runs_mongodump_with_username_and_password(): def test_dump_data_sources_runs_mongodump_with_directory_format(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'foo', 'format': 'directory'}] flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( @@ -189,6 +204,9 @@ def test_dump_data_sources_runs_mongodump_with_directory_format(): def test_dump_data_sources_runs_mongodump_with_options(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'foo', 'options': '--stuff=such'}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') @@ -222,6 +240,9 @@ def test_dump_data_sources_runs_mongodump_with_options(): def test_dump_data_sources_runs_mongodumpall_for_all_databases(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) databases = [{'name': 'all'}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') @@ -275,7 +296,7 @@ def test_build_dump_command_with_username_injection_attack_gets_escaped(): def test_restore_data_source_dump_runs_mongorestore(): hook_config = [{'name': 'foo', 'schemas': None}, {'name': 'bar'}] - extract_process = flexmock(stdout=flexmock()) + extract_process = flexmock(stdout=flexmock(read=lambda: b"")) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') @@ -290,9 +311,9 @@ def test_restore_data_source_dump_runs_mongorestore(): ).once() module.restore_data_source_dump( - hook_config, - {}, - data_source={'name': 'foo'}, + hook_config=hook_config, + config={}, + data_source=hook_config[0], dry_run=False, extract_process=extract_process, connection_params={ @@ -309,7 +330,7 @@ def test_restore_data_source_dump_runs_mongorestore_with_hostname_and_port(): hook_config = [ {'name': 'foo', 'hostname': 'database.example.org', 'port': 5433, 'schemas': None} ] - extract_process = flexmock(stdout=flexmock()) + extract_process = flexmock(stdout=flexmock(read=lambda: b"")) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') @@ -681,3 +702,105 @@ def test_restore_data_source_dump_without_extract_process_restores_from_disk(): }, borgmatic_runtime_directory='/run/borgmatic', ) +def test_dump_data_sources_uses_custom_mongodump_command(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) + databases = [{'name': 'foo', 'mongodump_command': 'custom_mongodump'}] + process = flexmock() + flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( + 'databases/localhost/foo' + ) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'custom_mongodump', + '--db', + 'foo', + '--archive', + '>', + 'databases/localhost/foo', + ), + shell=True, + run_to_completion=False, + ).and_return(process).once() + + assert module.dump_data_sources( + databases, + {}, + config_paths=('test.yaml',), + borgmatic_runtime_directory='/run/borgmatic', + patterns=[], + dry_run=False, + ) == [process] + +def test_build_dump_command_prevents_shell_injection(): + database = { + 'name': 'testdb; rm -rf /', # Malicious input + 'hostname': 'localhost', + 'port': 27017, + 'username': 'user', + 'password': 'password', + 'mongodump_command': 'mongodump', + 'options': '--gzip', + } + config = {} + dump_filename = '/path/to/dump' + dump_format = 'archive' + + from borgmatic.hooks.data_source.mongodb import build_dump_command, build_restore_command # Import the functions + + command = build_dump_command(database, config, dump_filename, dump_format) + + # Ensure the malicious input is properly escaped and does not execute + assert 'testdb; rm -rf /' not in command + assert any('testdb' in part for part in command) # Check if 'testdb' is in any part of the tuple + +def test_restore_data_source_dump_uses_custom_mongorestore_command(): + hook_config = [ + { + 'name': 'foo', + 'mongorestore_command': 'custom_mongorestore', + 'schemas': None, + 'restore_options': '--gzip', + } + ] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('make_dump_path') + flexmock(module.dump).should_receive('make_data_source_dump_filename') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('execute_command_with_processes').with_args( + [ + 'custom_mongorestore', # Should use custom command instead of default + '--archive', + '--drop', + '--gzip', # Should include restore options + ], + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + ).once() + + module.restore_data_source_dump( + hook_config, + {}, + data_source=hook_config[0], + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + borgmatic_runtime_directory='/run/borgmatic', + ) + + + + From 6a470be9241c3574f98375edde0f9534e6d9a6d8 Mon Sep 17 00:00:00 2001 From: Gautam Aggarwal Date: Mon, 24 Mar 2025 03:53:42 +0000 Subject: [PATCH 155/226] Made some changes in test file --- tests/unit/hooks/data_source/test_mongodb.py | 33 ++++---------------- 1 file changed, 6 insertions(+), 27 deletions(-) diff --git a/tests/unit/hooks/data_source/test_mongodb.py b/tests/unit/hooks/data_source/test_mongodb.py index bb8532c1..00f5fd9f 100644 --- a/tests/unit/hooks/data_source/test_mongodb.py +++ b/tests/unit/hooks/data_source/test_mongodb.py @@ -24,9 +24,6 @@ def test_use_streaming_false_for_no_databases(): def test_dump_data_sources_runs_mongodump_for_each_database(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'foo'}, {'name': 'bar'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') @@ -56,9 +53,6 @@ def test_dump_data_sources_runs_mongodump_for_each_database(): def test_dump_data_sources_with_dry_run_skips_mongodump(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'foo'}, {'name': 'bar'}] flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( @@ -81,9 +75,6 @@ def test_dump_data_sources_with_dry_run_skips_mongodump(): def test_dump_data_sources_runs_mongodump_with_hostname_and_port(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') @@ -120,12 +111,9 @@ def test_dump_data_sources_runs_mongodump_with_hostname_and_port(): def test_dump_data_sources_runs_mongodump_with_username_and_password(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [ { - 'name': 'foo', # Ensure this matches the expected format in the related functions + 'name': 'foo', 'username': 'mongo', 'password': 'trustsome1', 'authentication_database': 'admin', @@ -174,9 +162,6 @@ def test_dump_data_sources_runs_mongodump_with_username_and_password(): def test_dump_data_sources_runs_mongodump_with_directory_format(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'foo', 'format': 'directory'}] flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( @@ -204,9 +189,6 @@ def test_dump_data_sources_runs_mongodump_with_directory_format(): def test_dump_data_sources_runs_mongodump_with_options(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'foo', 'options': '--stuff=such'}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') @@ -240,9 +222,6 @@ def test_dump_data_sources_runs_mongodump_with_options(): def test_dump_data_sources_runs_mongodumpall_for_all_databases(): - flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( - flexmock() - ) databases = [{'name': 'all'}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') @@ -296,7 +275,7 @@ def test_build_dump_command_with_username_injection_attack_gets_escaped(): def test_restore_data_source_dump_runs_mongorestore(): hook_config = [{'name': 'foo', 'schemas': None}, {'name': 'bar'}] - extract_process = flexmock(stdout=flexmock(read=lambda: b"")) + extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') @@ -311,9 +290,9 @@ def test_restore_data_source_dump_runs_mongorestore(): ).once() module.restore_data_source_dump( - hook_config=hook_config, - config={}, - data_source=hook_config[0], + hook_config, + {}, + data_source={'name': 'foo'}, dry_run=False, extract_process=extract_process, connection_params={ @@ -330,7 +309,7 @@ def test_restore_data_source_dump_runs_mongorestore_with_hostname_and_port(): hook_config = [ {'name': 'foo', 'hostname': 'database.example.org', 'port': 5433, 'schemas': None} ] - extract_process = flexmock(stdout=flexmock(read=lambda: b"")) + extract_process = flexmock(stdout=flexmock()) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') From a1d2f7f2218f2bc44dcc147957c89f6844a9eaa8 Mon Sep 17 00:00:00 2001 From: Vandal <86826719+VandalByte@users.noreply.github.com> Date: Mon, 24 Mar 2025 11:51:33 +0530 Subject: [PATCH 156/226] add path --- borgmatic/borg/recreate.py | 4 ++++ borgmatic/commands/arguments.py | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/borgmatic/borg/recreate.py b/borgmatic/borg/recreate.py index a1bd60aa..fd547e2f 100644 --- a/borgmatic/borg/recreate.py +++ b/borgmatic/borg/recreate.py @@ -30,11 +30,15 @@ def recreate_archive( repo_archive_arg = make_repository_archive_flags(repository, archive, local_borg_version) exclude_flags = make_exclude_flags(config) + # handle path from recreate_arguments + path_flag = ('--path', recreate_arguments.path) if recreate_arguments.path else () + recreate_cmd = ( (local_path, 'recreate') + (('--remote-path', remote_path) if remote_path else ()) + repo_archive_arg + + path_flag + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 4018547e..7e942c63 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1562,6 +1562,13 @@ def make_parsers(): '--archive', help='Name of the archive to recreate', ) + recreate_group.add_argument( + '--path', + metavar='PATH', + dest='paths', + action='append', + help='Path to recreate the repository/archive', + ) recreate_group.add_argument( '-h', '--help', action='help', help='Show this help message and exit' ) From fa3b1405902940e20f22e7e8a796a2b6428aee11 Mon Sep 17 00:00:00 2001 From: Vandal <86826719+VandalByte@users.noreply.github.com> Date: Mon, 24 Mar 2025 12:09:08 +0530 Subject: [PATCH 157/226] add patterns --- borgmatic/actions/recreate.py | 8 ++++++++ borgmatic/borg/recreate.py | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/borgmatic/actions/recreate.py b/borgmatic/actions/recreate.py index acac0d44..0f3a0d79 100644 --- a/borgmatic/actions/recreate.py +++ b/borgmatic/actions/recreate.py @@ -2,6 +2,7 @@ import logging import borgmatic.borg.recreate import borgmatic.config.validate +from borgmatic.actions.create import collect_patterns, process_patterns logger = logging.getLogger(__name__) @@ -26,6 +27,12 @@ def run_recreate( else: logger.info('Recreating repository') + # collect and process patterns + patterns = collect_patterns(config) + processed_patterns = process_patterns( + patterns, borgmatic.config.paths.get_working_directory(config) + ) + borgmatic.borg.recreate.recreate_archive( repository['path'], borgmatic.borg.repo_list.resolve_archive_name( @@ -43,4 +50,5 @@ def run_recreate( global_arguments, local_path=local_path, remote_path=remote_path, + patterns=processed_patterns, ) diff --git a/borgmatic/borg/recreate.py b/borgmatic/borg/recreate.py index fd547e2f..9cc54679 100644 --- a/borgmatic/borg/recreate.py +++ b/borgmatic/borg/recreate.py @@ -18,6 +18,7 @@ def recreate_archive( global_arguments, local_path, remote_path=None, + patterns=None, ): ''' Given a local or remote repository path, an archive name, a configuration dict, @@ -32,7 +33,7 @@ def recreate_archive( exclude_flags = make_exclude_flags(config) # handle path from recreate_arguments path_flag = ('--path', recreate_arguments.path) if recreate_arguments.path else () - + pattern_flags = ('--patterns-from', patterns) if patterns else () recreate_cmd = ( (local_path, 'recreate') @@ -43,6 +44,7 @@ def recreate_archive( + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug', '--show-rc', '--list') if logger.isEnabledFor(logging.DEBUG) else ()) + + pattern_flags + exclude_flags ) From 4bca7bb1980e101871c32f157752caf34b4c41ad Mon Sep 17 00:00:00 2001 From: Nish_ <120EE0980@nitrkl.ac.in> Date: Mon, 24 Mar 2025 21:04:55 +0530 Subject: [PATCH 158/226] add directory expansion for file-based and KeyPassXC credentials Signed-off-by: Nish_ <120EE0980@nitrkl.ac.in> --- borgmatic/hooks/credential/file.py | 4 +- borgmatic/hooks/credential/keepassxc.py | 6 ++- tests/unit/hooks/credential/test_file.py | 28 ++++++++++++++ tests/unit/hooks/credential/test_keepassxc.py | 38 +++++++++++++++++++ 4 files changed, 73 insertions(+), 3 deletions(-) diff --git a/borgmatic/hooks/credential/file.py b/borgmatic/hooks/credential/file.py index 06854703..366a7d4a 100644 --- a/borgmatic/hooks/credential/file.py +++ b/borgmatic/hooks/credential/file.py @@ -19,9 +19,11 @@ def load_credential(hook_config, config, credential_parameters): raise ValueError(f'Cannot load invalid credential: "{name}"') + expanded_credential_path = os.path.expanduser(credential_path) + try: with open( - os.path.join(config.get('working_directory', ''), credential_path) + os.path.join(config.get('working_directory', ''), expanded_credential_path) ) as credential_file: return credential_file.read().rstrip(os.linesep) except (FileNotFoundError, OSError) as error: diff --git a/borgmatic/hooks/credential/keepassxc.py b/borgmatic/hooks/credential/keepassxc.py index e37be499..7e6ac913 100644 --- a/borgmatic/hooks/credential/keepassxc.py +++ b/borgmatic/hooks/credential/keepassxc.py @@ -24,7 +24,9 @@ def load_credential(hook_config, config, credential_parameters): f'Cannot load credential with invalid KeePassXC database path and attribute name: "{path_and_name}"' ) - if not os.path.exists(database_path): + expanded_database_path = os.path.expanduser(database_path) + + if not os.path.exists(expanded_database_path): raise ValueError( f'Cannot load credential because KeePassXC database path does not exist: {database_path}' ) @@ -36,7 +38,7 @@ def load_credential(hook_config, config, credential_parameters): '--show-protected', '--attributes', 'Password', - database_path, + expanded_database_path, attribute_name, ) ).rstrip(os.linesep) diff --git a/tests/unit/hooks/credential/test_file.py b/tests/unit/hooks/credential/test_file.py index aff10b89..42827a85 100644 --- a/tests/unit/hooks/credential/test_file.py +++ b/tests/unit/hooks/credential/test_file.py @@ -26,6 +26,9 @@ def test_load_credential_reads_named_credential_from_file(): credential_stream = io.StringIO('password') credential_stream.name = '/credentials/mycredential' builtins = flexmock(sys.modules['builtins']) + flexmock(module.os.path).should_receive('expanduser').with_args( + '/credentials/mycredential' + ).and_return('/credentials/mycredential') builtins.should_receive('open').with_args('/credentials/mycredential').and_return( credential_stream ) @@ -42,6 +45,9 @@ def test_load_credential_reads_named_credential_from_file_using_working_director credential_stream = io.StringIO('password') credential_stream.name = '/working/credentials/mycredential' builtins = flexmock(sys.modules['builtins']) + flexmock(module.os.path).should_receive('expanduser').with_args( + 'credentials/mycredential' + ).and_return('credentials/mycredential') builtins.should_receive('open').with_args('/working/credentials/mycredential').and_return( credential_stream ) @@ -58,6 +64,9 @@ def test_load_credential_reads_named_credential_from_file_using_working_director def test_load_credential_with_file_not_found_error_raises(): builtins = flexmock(sys.modules['builtins']) + flexmock(module.os.path).should_receive('expanduser').with_args( + '/credentials/mycredential' + ).and_return('/credentials/mycredential') builtins.should_receive('open').with_args('/credentials/mycredential').and_raise( FileNotFoundError ) @@ -66,3 +75,22 @@ def test_load_credential_with_file_not_found_error_raises(): module.load_credential( hook_config={}, config={}, credential_parameters=('/credentials/mycredential',) ) + + +def test_load_credential_reads_named_credential_from_expanded_directory(): + credential_stream = io.StringIO('password') + credential_stream.name = '/root/credentials/mycredential' + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os.path).should_receive('expanduser').with_args( + '~/credentials/mycredential' + ).and_return('/root/credentials/mycredential') + builtins.should_receive('open').with_args('/root/credentials/mycredential').and_return( + credential_stream + ) + + assert ( + module.load_credential( + hook_config={}, config={}, credential_parameters=('~/credentials/mycredential',) + ) + == 'password' + ) diff --git a/tests/unit/hooks/credential/test_keepassxc.py b/tests/unit/hooks/credential/test_keepassxc.py index e25c8717..0c460e23 100644 --- a/tests/unit/hooks/credential/test_keepassxc.py +++ b/tests/unit/hooks/credential/test_keepassxc.py @@ -15,6 +15,9 @@ def test_load_credential_with_invalid_credential_parameters_raises(credential_pa def test_load_credential_with_missing_database_raises(): + flexmock(module.os.path).should_receive('expanduser').with_args('database.kdbx').and_return( + 'database.kdbx' + ) flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.borgmatic.execute).should_receive('execute_command_and_capture_output').never() @@ -25,6 +28,9 @@ def test_load_credential_with_missing_database_raises(): def test_load_credential_with_present_database_fetches_password_from_keepassxc(): + flexmock(module.os.path).should_receive('expanduser').with_args('database.kdbx').and_return( + 'database.kdbx' + ) flexmock(module.os.path).should_receive('exists').and_return(True) flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' @@ -51,6 +57,9 @@ def test_load_credential_with_present_database_fetches_password_from_keepassxc() def test_load_credential_with_custom_keepassxc_cli_command_calls_it(): + flexmock(module.os.path).should_receive('expanduser').with_args('database.kdbx').and_return( + 'database.kdbx' + ) config = {'keepassxc': {'keepassxc_cli_command': '/usr/local/bin/keepassxc-cli --some-option'}} flexmock(module.os.path).should_receive('exists').and_return(True) flexmock(module.borgmatic.execute).should_receive( @@ -78,3 +87,32 @@ def test_load_credential_with_custom_keepassxc_cli_command_calls_it(): ) == 'password' ) + + +def test_load_credential_with_expanded_directory_with_present_database_fetches_password_from_keepassxc(): + flexmock(module.os.path).should_receive('expanduser').with_args('~/database.kdbx').and_return( + '/root/database.kdbx' + ) + flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).with_args( + ( + 'keepassxc-cli', + 'show', + '--show-protected', + '--attributes', + 'Password', + '/root/database.kdbx', + 'mypassword', + ) + ).and_return( + 'password' + ).once() + + assert ( + module.load_credential( + hook_config={}, config={}, credential_parameters=('~/database.kdbx', 'mypassword') + ) + == 'password' + ) From e7f14bca87724b0d74d8d74d4ab85a225cd1dc0e Mon Sep 17 00:00:00 2001 From: Vandal <86826719+VandalByte@users.noreply.github.com> Date: Tue, 25 Mar 2025 00:16:20 +0530 Subject: [PATCH 159/226] add tests and requested changes --- borgmatic/actions/recreate.py | 5 ++-- borgmatic/borg/recreate.py | 29 +++++++++++++------- tests/unit/actions/test_recreate.py | 39 +++++++++++++++++++++++++++ tests/unit/borg/test_recreate.py | 0 tests/unit/commands/test_borgmatic.py | 21 +++++++++++++++ 5 files changed, 82 insertions(+), 12 deletions(-) create mode 100644 tests/unit/actions/test_recreate.py create mode 100644 tests/unit/borg/test_recreate.py diff --git a/borgmatic/actions/recreate.py b/borgmatic/actions/recreate.py index 0f3a0d79..53ee7fce 100644 --- a/borgmatic/actions/recreate.py +++ b/borgmatic/actions/recreate.py @@ -27,10 +27,9 @@ def run_recreate( else: logger.info('Recreating repository') - # collect and process patterns - patterns = collect_patterns(config) + # Collect and process patterns. processed_patterns = process_patterns( - patterns, borgmatic.config.paths.get_working_directory(config) + collect_patterns(config), borgmatic.config.paths.get_working_directory(config) ) borgmatic.borg.recreate.recreate_archive( diff --git a/borgmatic/borg/recreate.py b/borgmatic/borg/recreate.py index 9cc54679..a3c2e7e4 100644 --- a/borgmatic/borg/recreate.py +++ b/borgmatic/borg/recreate.py @@ -3,8 +3,8 @@ import logging import borgmatic.borg.environment import borgmatic.config.paths import borgmatic.execute -from borgmatic.borg.create import make_exclude_flags -from borgmatic.borg.flags import make_flags_from_arguments, make_repository_archive_flags +from borgmatic.borg.create import make_exclude_flags, write_patterns_file, make_list_filter_flags +from borgmatic.borg.flags import make_repository_archive_flags logger = logging.getLogger(__name__) @@ -31,20 +31,31 @@ def recreate_archive( repo_archive_arg = make_repository_archive_flags(repository, archive, local_borg_version) exclude_flags = make_exclude_flags(config) - # handle path from recreate_arguments - path_flag = ('--path', recreate_arguments.path) if recreate_arguments.path else () - pattern_flags = ('--patterns-from', patterns) if patterns else () + + # Write patterns to a temporary file and use that file with --patterns-from. + patterns_file = write_patterns_file( + patterns, borgmatic.config.paths.get_working_directory(config) + ) recreate_cmd = ( (local_path, 'recreate') + (('--remote-path', remote_path) if remote_path else ()) + repo_archive_arg - + path_flag + + (('--path', recreate_arguments.path) if recreate_arguments.path else ()) + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) - + (('--debug', '--show-rc', '--list') if logger.isEnabledFor(logging.DEBUG) else ()) - + pattern_flags + + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + + (('--patterns-from', patterns_file.name) if patterns_file else ()) + + ( + ( + '--list', + '--filter', + make_list_filter_flags(local_borg_version, global_arguments.dry_run), + ) + if recreate_arguments.list + else () + ) + exclude_flags ) @@ -54,7 +65,7 @@ def recreate_archive( borgmatic.execute.execute_command( recreate_cmd, - output_log_level=logging.ANSWER, + output_log_level=logging.INFO, environment=borgmatic.borg.environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), remote_path=remote_path, diff --git a/tests/unit/actions/test_recreate.py b/tests/unit/actions/test_recreate.py new file mode 100644 index 00000000..4250af2c --- /dev/null +++ b/tests/unit/actions/test_recreate.py @@ -0,0 +1,39 @@ +from flexmock import flexmock + +from borgmatic.actions import recreate as module + + +def test_run_recreate_does_not_raise(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.recreate).should_receive('recreate_archive') + + recreate_arguments = flexmock(repository=flexmock(), archive=None) + + module.run_recreate( + repository={'path': 'repo'}, + config={}, + local_borg_version=None, + recreate_arguments=recreate_arguments, + global_arguments=flexmock(), + local_path=None, + remote_path=None, + ) + + +def test_run_recreate_with_archive_does_not_raise(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.recreate).should_receive('recreate_archive') + + recreate_arguments = flexmock(repository=flexmock(), archive='test-archive') + + module.run_recreate( + repository={'path': 'repo'}, + config={}, + local_borg_version=None, + recreate_arguments=recreate_arguments, + global_arguments=flexmock(), + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/borg/test_recreate.py b/tests/unit/borg/test_recreate.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 160f4454..0c474c7d 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -1019,6 +1019,27 @@ def test_run_actions_runs_create(): assert result == (expected,) +def test_run_actions_runs_recreate(): + flexmock(module).should_receive('add_custom_log_levels') + flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) + + flexmock(borgmatic.actions.recreate).should_receive('run_recreate').once() + + tuple( + module.run_actions( + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'recreate': flexmock()}, + config_filename=flexmock(), + config={'repositories': []}, + config_paths=[], + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), + repository={'path': 'repo'}, + ) + ) + + def test_run_actions_with_skip_actions_skips_create(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return(['create']) From b60cf2449afa1f316230b86773e4435163949a26 Mon Sep 17 00:00:00 2001 From: Vandal <86826719+VandalByte@users.noreply.github.com> Date: Tue, 25 Mar 2025 00:48:27 +0530 Subject: [PATCH 160/226] add recreate to schema --- borgmatic/config/schema.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 7b309ea0..9fb8f361 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -767,6 +767,7 @@ properties: - prune - compact - create + - recreate - check - delete - extract @@ -982,6 +983,7 @@ properties: - prune - compact - create + - recreate - check - delete - extract @@ -1046,6 +1048,7 @@ properties: - prune - compact - create + - recreate - check - delete - extract From e1fdfe4c2ff0eec35f40d1ee5ece25696c671043 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 24 Mar 2025 13:00:38 -0700 Subject: [PATCH 161/226] Add credential hook directory expansion to NEWS (#422). --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index a7da35bc..364b86df 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,6 @@ 2.0.0.dev0 * #345: Add a "key import" action to import a repository key from backup. + * #422: Add home directory expansion to file-based and KeePassXC credential hooks. * #790, #821: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more flexible "commands:". See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ From 532a97623cda78bd93c1a15a2b36519ca189b8a6 Mon Sep 17 00:00:00 2001 From: Gautam Aggarwal Date: Tue, 25 Mar 2025 04:50:45 +0000 Subject: [PATCH 162/226] Added test_build_restore_command_prevents_shell_injection() --- tests/unit/hooks/data_source/test_mongodb.py | 34 +++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/tests/unit/hooks/data_source/test_mongodb.py b/tests/unit/hooks/data_source/test_mongodb.py index 00f5fd9f..7ae036c9 100644 --- a/tests/unit/hooks/data_source/test_mongodb.py +++ b/tests/unit/hooks/data_source/test_mongodb.py @@ -729,9 +729,7 @@ def test_build_dump_command_prevents_shell_injection(): dump_filename = '/path/to/dump' dump_format = 'archive' - from borgmatic.hooks.data_source.mongodb import build_dump_command, build_restore_command # Import the functions - - command = build_dump_command(database, config, dump_filename, dump_format) + command = module.build_dump_command(database, config, dump_filename, dump_format) # Ensure the malicious input is properly escaped and does not execute assert 'testdb; rm -rf /' not in command @@ -779,7 +777,35 @@ def test_restore_data_source_dump_uses_custom_mongorestore_command(): }, borgmatic_runtime_directory='/run/borgmatic', ) + +def test_build_restore_command_prevents_shell_injection(): + database = { + 'name': 'testdb; rm -rf /', # Malicious input + 'restore_hostname': 'localhost', + 'restore_port': 27017, + 'restore_username': 'user', + 'restore_password': 'password', + 'mongorestore_command': 'mongorestore', + 'restore_options': '--gzip', + } + config = {} + dump_filename = '/path/to/dump' + connection_params = { + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + } + extract_process = None + + command = module.build_restore_command( + extract_process, database, config, dump_filename, connection_params + ) + + # print(command) + # Ensure the malicious input is properly escaped and does not execute + assert 'rm -rf /' not in command + assert ';' not in command - From 354267344601e831b9ebf973786f3c4e631c5f1c Mon Sep 17 00:00:00 2001 From: Vandal <86826719+VandalByte@users.noreply.github.com> Date: Tue, 25 Mar 2025 11:36:06 +0530 Subject: [PATCH 163/226] add test recreate with skip action --- tests/unit/commands/test_borgmatic.py | 30 ++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 0c474c7d..481a856a 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -1019,6 +1019,26 @@ def test_run_actions_runs_create(): assert result == (expected,) +def test_run_actions_with_skip_actions_skips_create(): + flexmock(module).should_receive('add_custom_log_levels') + flexmock(module).should_receive('get_skip_actions').and_return(['create']) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) + flexmock(borgmatic.actions.create).should_receive('run_create').never() + + tuple( + module.run_actions( + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'create': flexmock()}, + config_filename=flexmock(), + config={'repositories': [], 'skip_actions': ['create']}, + config_paths=[], + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), + repository={'path': 'repo'}, + ) + ) + + def test_run_actions_runs_recreate(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) @@ -1040,17 +1060,17 @@ def test_run_actions_runs_recreate(): ) -def test_run_actions_with_skip_actions_skips_create(): +def test_run_actions_with_skip_actions_skips_recreate(): flexmock(module).should_receive('add_custom_log_levels') - flexmock(module).should_receive('get_skip_actions').and_return(['create']) + flexmock(module).should_receive('get_skip_actions').and_return(['recreate']) flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) - flexmock(borgmatic.actions.create).should_receive('run_create').never() + flexmock(borgmatic.actions.recreate).should_receive('run_recreate').never() tuple( module.run_actions( - arguments={'global': flexmock(dry_run=False, log_file='foo'), 'create': flexmock()}, + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'recreate': flexmock()}, config_filename=flexmock(), - config={'repositories': [], 'skip_actions': ['create']}, + config={'repositories': [], 'skip_actions': ['recreate']}, config_paths=[], local_path=flexmock(), remote_path=flexmock(), From a8726c408aa5a6cc4920649081a84da4f4365c5a Mon Sep 17 00:00:00 2001 From: Vandal <86826719+VandalByte@users.noreply.github.com> Date: Tue, 25 Mar 2025 19:35:15 +0530 Subject: [PATCH 164/226] add tests --- borgmatic/borg/recreate.py | 5 +-- borgmatic/commands/borgmatic.py | 2 +- tests/unit/borg/test_recreate.py | 64 ++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/borgmatic/borg/recreate.py b/borgmatic/borg/recreate.py index a3c2e7e4..8a85908c 100644 --- a/borgmatic/borg/recreate.py +++ b/borgmatic/borg/recreate.py @@ -3,7 +3,7 @@ import logging import borgmatic.borg.environment import borgmatic.config.paths import borgmatic.execute -from borgmatic.borg.create import make_exclude_flags, write_patterns_file, make_list_filter_flags +from borgmatic.borg.create import make_exclude_flags, make_list_filter_flags, write_patterns_file from borgmatic.borg.flags import make_repository_archive_flags logger = logging.getLogger(__name__) @@ -64,11 +64,10 @@ def recreate_archive( return borgmatic.execute.execute_command( - recreate_cmd, + full_command=recreate_cmd, output_log_level=logging.INFO, environment=borgmatic.borg.environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), - remote_path=remote_path, borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), ) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 467673fe..53d9582c 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -17,7 +17,6 @@ import borgmatic.actions.config.bootstrap import borgmatic.actions.config.generate import borgmatic.actions.config.validate import borgmatic.actions.create -import borgmatic.actions.recreate import borgmatic.actions.delete import borgmatic.actions.export_key import borgmatic.actions.export_tar @@ -27,6 +26,7 @@ import borgmatic.actions.info import borgmatic.actions.list import borgmatic.actions.mount import borgmatic.actions.prune +import borgmatic.actions.recreate import borgmatic.actions.repo_create import borgmatic.actions.repo_delete import borgmatic.actions.repo_info diff --git a/tests/unit/borg/test_recreate.py b/tests/unit/borg/test_recreate.py index e69de29b..8e229eea 100644 --- a/tests/unit/borg/test_recreate.py +++ b/tests/unit/borg/test_recreate.py @@ -0,0 +1,64 @@ +# import logging + +from flexmock import flexmock + +from borgmatic.borg import recreate as module + +# from ..test_verbosity import insert_logging_mock + +# from borgmatic.borg.pattern import Pattern, Pattern_type, Pattern_style, Pattern_source +# from borgmatic.borg.create import make_exclude_flags, make_list_filter_flags + + +def insert_execute_command_mock(command, working_directory=None, borg_exit_codes=None): + flexmock(module.borgmatic.borg.environment).should_receive('make_environment') + flexmock(module.borgmatic.execute).should_receive('execute_command').with_args( + full_command=command, + output_log_level=module.logging.INFO, + environment=None, + working_directory=working_directory, + borg_local_path=command[0], + borg_exit_codes=borg_exit_codes, + ).once() + + +def test_recreate_archive_dry_run_skips_execution(): + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) + flexmock(module.borgmatic.execute).should_receive('execute_command').never() + + recreate_arguments = flexmock(repository=flexmock(), list=None, path=None) + + result = module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=recreate_arguments, + global_arguments=flexmock(log_json=False, dry_run=True), + local_path='borg', + ) + + assert result is None + + +def test_recreate_calls_borg_with_required_flags(): + flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( + ('repo::archive',) + ) + flexmock(module).should_receive('write_patterns_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return(None) + insert_execute_command_mock(('borg', 'recreate', 'repo::archive')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock(path=None, list=None), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + remote_path=None, + patterns=None, + ) From ea5a2d8a462cef1b8eb39d485306d6da73b41a0e Mon Sep 17 00:00:00 2001 From: Vandal <86826719+VandalByte@users.noreply.github.com> Date: Tue, 25 Mar 2025 20:39:02 +0530 Subject: [PATCH 165/226] add tests for the flags --- borgmatic/borg/recreate.py | 2 +- tests/unit/borg/test_recreate.py | 196 ++++++++++++++++++++++++++++++- 2 files changed, 195 insertions(+), 3 deletions(-) diff --git a/borgmatic/borg/recreate.py b/borgmatic/borg/recreate.py index 8a85908c..a2a45047 100644 --- a/borgmatic/borg/recreate.py +++ b/borgmatic/borg/recreate.py @@ -40,7 +40,6 @@ def recreate_archive( recreate_cmd = ( (local_path, 'recreate') + (('--remote-path', remote_path) if remote_path else ()) - + repo_archive_arg + (('--path', recreate_arguments.path) if recreate_arguments.path else ()) + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) @@ -57,6 +56,7 @@ def recreate_archive( else () ) + exclude_flags + + repo_archive_arg ) if global_arguments.dry_run: diff --git a/tests/unit/borg/test_recreate.py b/tests/unit/borg/test_recreate.py index 8e229eea..379ecc3a 100644 --- a/tests/unit/borg/test_recreate.py +++ b/tests/unit/borg/test_recreate.py @@ -1,10 +1,10 @@ -# import logging +import logging from flexmock import flexmock from borgmatic.borg import recreate as module -# from ..test_verbosity import insert_logging_mock +from ..test_verbosity import insert_logging_mock # from borgmatic.borg.pattern import Pattern, Pattern_type, Pattern_style, Pattern_source # from borgmatic.borg.create import make_exclude_flags, make_list_filter_flags @@ -62,3 +62,195 @@ def test_recreate_calls_borg_with_required_flags(): remote_path=None, patterns=None, ) + + +def test_recreate_with_remote_path(): + flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( + ('repo::archive',) + ) + flexmock(module).should_receive('write_patterns_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return(None) + insert_execute_command_mock(('borg', 'recreate', '--remote-path', 'borg1', 'repo::archive')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock(path=None, list=None), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + remote_path='borg1', + patterns=None, + ) + + +def test_recreate_with_lock_wait(): + flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( + ('repo::archive',) + ) + flexmock(module).should_receive('write_patterns_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return(None) + insert_execute_command_mock(('borg', 'recreate', '--lock-wait', '5', 'repo::archive')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={'lock_wait': '5'}, + local_borg_version='1.2.3', + recreate_arguments=flexmock(path=None, list=None), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_log_info(): + flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( + ('repo::archive',) + ) + flexmock(module).should_receive('write_patterns_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return(None) + insert_execute_command_mock(('borg', 'recreate', '--info', 'repo::archive')) + + insert_logging_mock(logging.INFO) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock(path=None, list=None), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_log_debug(): + flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( + ('repo::archive',) + ) + flexmock(module).should_receive('write_patterns_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return(None) + insert_execute_command_mock(('borg', 'recreate', '--debug', '--show-rc', 'repo::archive')) + insert_logging_mock(logging.DEBUG) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock(path=None, list=None), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_log_json(): + flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( + ('repo::archive',) + ) + flexmock(module).should_receive('write_patterns_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return(None) + insert_execute_command_mock(('borg', 'recreate', '--log-json', 'repo::archive')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock(path=None, list=None), + global_arguments=flexmock(dry_run=False, log_json=True), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_path_flag(): + flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( + ('repo::archive',) + ) + flexmock(module).should_receive('write_patterns_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return(None) + insert_execute_command_mock(('borg', 'recreate', '--path', '/some/path', 'repo::archive')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock(path='/some/path', list=None), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_list_filter_flags(): + flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( + ('repo::archive',) + ) + flexmock(module).should_receive('write_patterns_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return('AME+-') + insert_execute_command_mock( + ('borg', 'recreate', '--list', '--filter', 'AME+-', 'repo::archive') + ) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock(path=None, list=True), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_patterns_from_flag(): + flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( + ('repo::archive',) + ) + mock_patterns_file = flexmock(name='patterns_file') + flexmock(module).should_receive('write_patterns_file').and_return(mock_patterns_file) + flexmock(module).should_receive('make_list_filter_flags').and_return(None) + insert_execute_command_mock( + ('borg', 'recreate', '--patterns-from', 'patterns_file', 'repo::archive') + ) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock(path=None, list=None), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=['pattern1', 'pattern2'], + ) + + +def test_recreate_with_exclude_flags(): + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) + flexmock(module).should_receive('write_patterns_file').and_return(None) + flexmock(module).should_receive('make_list_filter_flags').and_return(None) + # Mock the make_exclude_flags to return a sample exclude flag + flexmock(module).should_receive('make_exclude_flags').and_return(('--exclude', 'pattern')) + + insert_execute_command_mock(('borg', 'recreate', '--exclude', 'pattern', 'repo::archive')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={'exclude_patterns': ['pattern']}, + local_borg_version='1.2.3', + recreate_arguments=flexmock(path=None, list=None), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) From 203e84b91f17e743a249dfa2b52a44fb6c8173af Mon Sep 17 00:00:00 2001 From: Vandal <86826719+VandalByte@users.noreply.github.com> Date: Tue, 25 Mar 2025 21:57:06 +0530 Subject: [PATCH 166/226] hotfix --- borgmatic/borg/recreate.py | 14 +++++++++++--- tests/unit/borg/test_recreate.py | 18 ++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/borgmatic/borg/recreate.py b/borgmatic/borg/recreate.py index a2a45047..bd35651f 100644 --- a/borgmatic/borg/recreate.py +++ b/borgmatic/borg/recreate.py @@ -29,6 +29,10 @@ def recreate_archive( ''' lock_wait = config.get('lock_wait', None) + if archive is None: + logger.error('Please provide a valid archive name.') + return + repo_archive_arg = make_repository_archive_flags(repository, archive, local_borg_version) exclude_flags = make_exclude_flags(config) @@ -40,9 +44,13 @@ def recreate_archive( recreate_cmd = ( (local_path, 'recreate') + (('--remote-path', remote_path) if remote_path else ()) - + (('--path', recreate_arguments.path) if recreate_arguments.path else ()) + + ( + ('--path', recreate_arguments.path) + if hasattr(recreate_arguments, 'path') and recreate_arguments.path + else () + ) + (('--log-json',) if global_arguments.log_json else ()) - + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + + (('--lock-wait', str(lock_wait)) if lock_wait is not None else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (('--patterns-from', patterns_file.name) if patterns_file else ()) @@ -52,7 +60,7 @@ def recreate_archive( '--filter', make_list_filter_flags(local_borg_version, global_arguments.dry_run), ) - if recreate_arguments.list + if hasattr(recreate_arguments, 'list') and recreate_arguments.list else () ) + exclude_flags diff --git a/tests/unit/borg/test_recreate.py b/tests/unit/borg/test_recreate.py index 379ecc3a..4bab00d9 100644 --- a/tests/unit/borg/test_recreate.py +++ b/tests/unit/borg/test_recreate.py @@ -64,6 +64,24 @@ def test_recreate_calls_borg_with_required_flags(): ) +def test_recreate_calls_borg_without_archive(): + logger_mock = flexmock(module.logger) + logger_mock.should_receive('error').with_args('Please provide a valid archive name.').once() + + flexmock(module.borgmatic.execute).should_receive('execute_command').never() + + module.recreate_archive( + repository='repo', + archive=None, + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock(path=None, list=None), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + def test_recreate_with_remote_path(): flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( ('repo::archive',) From 9e694e4df9735f0b504d0a51bfae4c26f0448138 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 25 Mar 2025 22:28:14 -0700 Subject: [PATCH 167/226] Add MongoDB custom command options to NEWS (#837). --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index 364b86df..fd41e40c 100644 --- a/NEWS +++ b/NEWS @@ -10,6 +10,7 @@ * #790: BREAKING: Run all command hooks (both new and deprecated) respecting the "working_directory" option if configured, meaning that hook commands are run in that directory. * #836: Add a custom command option for the SQLite hook. + * #837: Add custom command options for the MongoDB hook. * #1010: When using Borg 2, don't pass the "--stats" flag to "borg prune". * #1020: Document a database use case involving a temporary database client container: https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers From 23efbb8df36ba81bf8e00f95100f390c9f51e3b0 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 25 Mar 2025 22:31:50 -0700 Subject: [PATCH 168/226] Fix line wrapping / code style (#837). --- borgmatic/config/schema.yaml | 15 ++++++++------- tests/unit/hooks/data_source/test_mongodb.py | 20 ++++++++++++-------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index ffbde053..bb0dc794 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -1729,20 +1729,21 @@ properties: mongodump_command: type: string description: | - Command to use instead of "mongodump". This can be used to - run a specific mongodump version (e.g., one inside a + Command to use instead of "mongodump". This can be used + to run a specific mongodump version (e.g., one inside a running container). If you run it from within a container, make sure to mount the path in the "user_runtime_directory" option from the host into the - container at the same location. Defaults to "mongodump". + container at the same location. Defaults to + "mongodump". example: docker exec mongodb_container mongodump mongorestore_command: type: string description: | - Command to run when restoring a database instead - of "mongorestore". This can be used to run a specific - mongorestore version (e.g., one inside a running container). - Defaults to "mongorestore". + Command to run when restoring a database instead of + "mongorestore". This can be used to run a specific + mongorestore version (e.g., one inside a running + container). Defaults to "mongorestore". example: docker exec mongodb_container mongorestore description: | List of one or more MongoDB databases to dump before creating a diff --git a/tests/unit/hooks/data_source/test_mongodb.py b/tests/unit/hooks/data_source/test_mongodb.py index 7ae036c9..3cd0aff3 100644 --- a/tests/unit/hooks/data_source/test_mongodb.py +++ b/tests/unit/hooks/data_source/test_mongodb.py @@ -681,6 +681,8 @@ def test_restore_data_source_dump_without_extract_process_restores_from_disk(): }, borgmatic_runtime_directory='/run/borgmatic', ) + + def test_dump_data_sources_uses_custom_mongodump_command(): flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( flexmock() @@ -714,7 +716,8 @@ def test_dump_data_sources_uses_custom_mongodump_command(): patterns=[], dry_run=False, ) == [process] - + + def test_build_dump_command_prevents_shell_injection(): database = { 'name': 'testdb; rm -rf /', # Malicious input @@ -733,8 +736,11 @@ def test_build_dump_command_prevents_shell_injection(): # Ensure the malicious input is properly escaped and does not execute assert 'testdb; rm -rf /' not in command - assert any('testdb' in part for part in command) # Check if 'testdb' is in any part of the tuple - + assert any( + 'testdb' in part for part in command + ) # Check if 'testdb' is in any part of the tuple + + def test_restore_data_source_dump_uses_custom_mongorestore_command(): hook_config = [ { @@ -777,7 +783,8 @@ def test_restore_data_source_dump_uses_custom_mongorestore_command(): }, borgmatic_runtime_directory='/run/borgmatic', ) - + + def test_build_restore_command_prevents_shell_injection(): database = { 'name': 'testdb; rm -rf /', # Malicious input @@ -801,11 +808,8 @@ def test_build_restore_command_prevents_shell_injection(): command = module.build_restore_command( extract_process, database, config, dump_filename, connection_params ) - + # print(command) # Ensure the malicious input is properly escaped and does not execute assert 'rm -rf /' not in command assert ';' not in command - - - From d2714cb706c45ccc05f4890bedfa2d97e4e194ee Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 26 Mar 2025 09:53:52 -0700 Subject: [PATCH 169/226] Fix an error in the systemd credential hook when the credential name contains a "." chararcter (#1044). --- NEWS | 2 ++ borgmatic/hooks/credential/systemd.py | 2 +- tests/unit/hooks/credential/test_systemd.py | 6 +++--- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index fd41e40c..3ae8b8e6 100644 --- a/NEWS +++ b/NEWS @@ -16,6 +16,8 @@ https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers * #1037: Fix an error with the "extract" action when both a remote repository and a "working_directory" are used. + * #1044: Fix an error in the systemd credential hook when the credential name contains a "." + chararcter. 1.9.14 * #409: With the PagerDuty monitoring hook, send borgmatic logs to PagerDuty so they show up in the diff --git a/borgmatic/hooks/credential/systemd.py b/borgmatic/hooks/credential/systemd.py index 1de245a3..9cce0d7b 100644 --- a/borgmatic/hooks/credential/systemd.py +++ b/borgmatic/hooks/credential/systemd.py @@ -5,7 +5,7 @@ import re logger = logging.getLogger(__name__) -CREDENTIAL_NAME_PATTERN = re.compile(r'^\w+$') +CREDENTIAL_NAME_PATTERN = re.compile(r'^[\w.-]+$') def load_credential(hook_config, config, credential_parameters): diff --git a/tests/unit/hooks/credential/test_systemd.py b/tests/unit/hooks/credential/test_systemd.py index 603c8569..32133883 100644 --- a/tests/unit/hooks/credential/test_systemd.py +++ b/tests/unit/hooks/credential/test_systemd.py @@ -42,12 +42,12 @@ def test_load_credential_reads_named_credential_from_file(): '/var' ) credential_stream = io.StringIO('password') - credential_stream.name = '/var/mycredential' + credential_stream.name = '/var/borgmatic.pw' builtins = flexmock(sys.modules['builtins']) - builtins.should_receive('open').with_args('/var/mycredential').and_return(credential_stream) + builtins.should_receive('open').with_args('/var/borgmatic.pw').and_return(credential_stream) assert ( - module.load_credential(hook_config={}, config={}, credential_parameters=('mycredential',)) + module.load_credential(hook_config={}, config={}, credential_parameters=('borgmatic.pw',)) == 'password' ) From 79e4e089eeffd097472c2ccb8aacc03a54ba0fba Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 26 Mar 2025 09:57:53 -0700 Subject: [PATCH 170/226] Fix typo in NEWS (#1044). --- NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 3ae8b8e6..df2261d8 100644 --- a/NEWS +++ b/NEWS @@ -17,7 +17,7 @@ * #1037: Fix an error with the "extract" action when both a remote repository and a "working_directory" are used. * #1044: Fix an error in the systemd credential hook when the credential name contains a "." - chararcter. + character. 1.9.14 * #409: With the PagerDuty monitoring hook, send borgmatic logs to PagerDuty so they show up in the From 93569244189e119a7aa12193b939dd543c195022 Mon Sep 17 00:00:00 2001 From: Vandal <86826719+VandalByte@users.noreply.github.com> Date: Wed, 26 Mar 2025 22:30:11 +0530 Subject: [PATCH 171/226] add archive options --- borgmatic/borg/recreate.py | 34 +++++++++++++++++---------------- borgmatic/commands/arguments.py | 34 ++++++++++++++++++++++++++------- 2 files changed, 45 insertions(+), 23 deletions(-) diff --git a/borgmatic/borg/recreate.py b/borgmatic/borg/recreate.py index bd35651f..fcf5ac0c 100644 --- a/borgmatic/borg/recreate.py +++ b/borgmatic/borg/recreate.py @@ -4,7 +4,7 @@ import borgmatic.borg.environment import borgmatic.config.paths import borgmatic.execute from borgmatic.borg.create import make_exclude_flags, make_list_filter_flags, write_patterns_file -from borgmatic.borg.flags import make_repository_archive_flags +from borgmatic.borg import flags logger = logging.getLogger(__name__) @@ -28,27 +28,22 @@ def recreate_archive( Executes the recreate command with the given arguments. ''' lock_wait = config.get('lock_wait', None) - - if archive is None: - logger.error('Please provide a valid archive name.') - return - - repo_archive_arg = make_repository_archive_flags(repository, archive, local_borg_version) exclude_flags = make_exclude_flags(config) + compression = config.get('compression', None) # Write patterns to a temporary file and use that file with --patterns-from. patterns_file = write_patterns_file( patterns, borgmatic.config.paths.get_working_directory(config) ) - recreate_cmd = ( + recreate_command = ( (local_path, 'recreate') + (('--remote-path', remote_path) if remote_path else ()) - + ( - ('--path', recreate_arguments.path) - if hasattr(recreate_arguments, 'path') and recreate_arguments.path - else () - ) + # + ( + # ('--path', recreate_arguments.path) + # if recreate_arguments.path + # else () + # ) + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait is not None else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) @@ -60,11 +55,18 @@ def recreate_archive( '--filter', make_list_filter_flags(local_borg_version, global_arguments.dry_run), ) - if hasattr(recreate_arguments, 'list') and recreate_arguments.list + if recreate_arguments.list else () ) + + (('--target', recreate_arguments.target) if recreate_arguments.target else ()) + + (('--comment', f'"{recreate_arguments.comment}"') if recreate_arguments.comment else ()) + + (('--compression', compression) if compression else ()) + exclude_flags - + repo_archive_arg + + ( + flags.make_repository_archive_flags(repository, archive, local_borg_version) + if archive + else flags.make_repository_flags(repository, local_borg_version) + ) ) if global_arguments.dry_run: @@ -72,7 +74,7 @@ def recreate_archive( return borgmatic.execute.execute_command( - full_command=recreate_cmd, + full_command=recreate_command, output_log_level=logging.INFO, environment=borgmatic.borg.environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 7e942c63..8c3d7c6f 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1556,18 +1556,38 @@ def make_parsers(): recreate_group = recreate_parser.add_argument_group('recreate arguments') recreate_group.add_argument( '--repository', - help='Path of the repository containing the archive', + help='Path of repository containing archive to recreate, defaults to the configured repository if there is only one, quoted globs supported', ) recreate_group.add_argument( '--archive', - help='Name of the archive to recreate', + help='Archive name, hash, or series to recreate', + ) + # recreate_group.add_argument( + # '--path', + # metavar='PATH', + # dest='path', + # help='Path to recreate the repository/archive', + # required=True, + # ) + recreate_group.add_argument( + '--list', dest='list', action='store_true', help='Show per-file details' ) recreate_group.add_argument( - '--path', - metavar='PATH', - dest='paths', - action='append', - help='Path to recreate the repository/archive', + '--target', + metavar='TARGET', + help='Name of new archive name', + ) + recreate_group.add_argument( + '--comment', + metavar='COMMENT', + help='Add a comment text to the archive, if archive not provided, consider all archives', + ) + recreate_group.add_argument( + '--compression', + '-C', + dest='compression', + metavar='COMPRESSION', + help='Select compression algorithm', ) recreate_group.add_argument( '-h', '--help', action='help', help='Show this help message and exit' From 996b03794669b6a96ce4c60522ab856e6d65a713 Mon Sep 17 00:00:00 2001 From: Gautam Aggarwal Date: Wed, 26 Mar 2025 17:39:10 +0000 Subject: [PATCH 172/226] 1st --- borgmatic/commands/borgmatic.py | 19 +++++++++++++++++++ borgmatic/config/schema.yaml | 6 ++++++ 2 files changed, 25 insertions(+) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index bd1ec1e6..d8c27034 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -943,6 +943,22 @@ def exit_with_help_link(): # pragma: no cover sys.exit(1) +def check_and_show_help_on_no_args(): + """ + Check if the 'borgmatic' command is run without any arguments. If the configuration option + 'show_help_on_no_args' is set to True, show the help message. Otherwise, trigger the + default backup behavior. + """ + if len(sys.argv) == 1: # No arguments provided + show_help_on_no_args = any( + config.get('show_help_on_no_args', False) + for config in load_configurations(tuple(collect.collect_config_filenames(None)))[0].values() + ) + if show_help_on_no_args: + print(parse_arguments('--help')) + sys.exit(0) + + def main(extra_summary_logs=[]): # pragma: no cover configure_signals() configure_delayed_logging() @@ -971,6 +987,9 @@ def main(extra_summary_logs=[]): # pragma: no cover print(borgmatic.commands.completion.fish.fish_completion()) sys.exit(0) + # Use the helper function to check and show help on no arguments + check_and_show_help_on_no_args() + validate = bool('validate' in arguments) config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths)) configs, config_paths, parse_logs = load_configurations( diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index ffbde053..96bba3b9 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -2665,3 +2665,9 @@ properties: example: /usr/local/bin/keepassxc-cli description: | Configuration for integration with the KeePassXC password manager. + show_help_on_no_args: + type: boolean + description: | + If true, running borgmatic without any arguments will display the + help message instead of triggering a backup. Defaults to false. + example: true From acc2814f117f0c884bd3186d93e9f6cf2df71916 Mon Sep 17 00:00:00 2001 From: Vandal <86826719+VandalByte@users.noreply.github.com> Date: Wed, 26 Mar 2025 23:39:06 +0530 Subject: [PATCH 173/226] add archive timestamp filter --- borgmatic/borg/recreate.py | 25 +++++++++++++++++++++++-- borgmatic/commands/arguments.py | 7 ++++++- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/borgmatic/borg/recreate.py b/borgmatic/borg/recreate.py index fcf5ac0c..640bcd05 100644 --- a/borgmatic/borg/recreate.py +++ b/borgmatic/borg/recreate.py @@ -1,14 +1,24 @@ import logging +from datetime import datetime import borgmatic.borg.environment import borgmatic.config.paths import borgmatic.execute -from borgmatic.borg.create import make_exclude_flags, make_list_filter_flags, write_patterns_file from borgmatic.borg import flags +from borgmatic.borg.create import make_exclude_flags, make_list_filter_flags, write_patterns_file logger = logging.getLogger(__name__) +def is_valid_timestamp(time_str): + try: + # Attempt to parse the time string using the expected format + datetime.strptime(time_str, r"%Y-%m-%dT%H:%M:%S") + return True + except ValueError: + return False + + def recreate_archive( repository, archive, @@ -27,6 +37,12 @@ def recreate_archive( Executes the recreate command with the given arguments. ''' + if recreate_arguments.timestamp and not is_valid_timestamp(recreate_arguments.timestamp): + logger.error( + 'Please provide a valid timestamp of format: yyyy-mm-ddThh:mm:ss. Example: 2025-03-26T14:45:59' + ) + return + lock_wait = config.get('lock_wait', None) exclude_flags = make_exclude_flags(config) compression = config.get('compression', None) @@ -58,7 +74,12 @@ def recreate_archive( if recreate_arguments.list else () ) - + (('--target', recreate_arguments.target) if recreate_arguments.target else ()) + # Flag --target works only for a single archive + + ( + ('--target', recreate_arguments.target) + if recreate_arguments.target and recreate_arguments.archive + else () + ) + (('--comment', f'"{recreate_arguments.comment}"') if recreate_arguments.comment else ()) + (('--compression', compression) if compression else ()) + exclude_flags diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 8c3d7c6f..f9d0e7af 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1587,7 +1587,12 @@ def make_parsers(): '-C', dest='compression', metavar='COMPRESSION', - help='Select compression algorithm', + help='Select the compression algorithm', + ) + recreate_group.add_argument( + '--timestamp', + metavar='TIMESTAMP', + help='Manually specify the archive creation date/time (UTC, yyyy-mm-ddThh:mm:ss format)', ) recreate_group.add_argument( '-h', '--help', action='help', help='Show this help message and exit' From 520fb78a00fb4e736e71a6973c9e439f36a5376b Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 26 Mar 2025 11:39:16 -0700 Subject: [PATCH 174/226] Clarify Btrfs documentation: borgmatic expects subvolume mount points in "source_directories" (#1043). --- docs/how-to/snapshot-your-filesystems.md | 32 ++++++++++++------------ 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/how-to/snapshot-your-filesystems.md b/docs/how-to/snapshot-your-filesystems.md index 8dd8bad5..018e6553 100644 --- a/docs/how-to/snapshot-your-filesystems.md +++ b/docs/how-to/snapshot-your-filesystems.md @@ -148,9 +148,9 @@ feedback](https://torsion.org/borgmatic/#issues) you have on this feature. #### Subvolume discovery -For any read-write subvolume you'd like backed up, add its path to borgmatic's -`source_directories` option. Btrfs does not support snapshotting read-only -subvolumes. +For any read-write subvolume you'd like backed up, add its mount point path to +borgmatic's `source_directories` option. Btrfs does not support snapshotting +read-only subvolumes. New in version 1.9.6 Or include the mount point as a root pattern with borgmatic's `patterns` or `patterns_from` @@ -161,27 +161,27 @@ includes the snapshotted files in the paths sent to Borg. borgmatic is also responsible for cleaning up (deleting) these snapshots after a backup completes. borgmatic is smart enough to look at the parent (and grandparent, etc.) -directories of each of your `source_directories` to discover any subvolumes. -For instance, let's say you add `/var/log` and `/var/lib` to your source -directories, but `/var` is a subvolume. borgmatic will discover that and -snapshot `/var` accordingly. This also works even with nested subvolumes; +directories of each of your `source_directories` to discover any subvolumes. For +instance, let's say you add `/var/log` and `/var/lib` to your source +directories, but `/var` is a subvolume mount point. borgmatic will discover that +and snapshot `/var` accordingly. This also works even with nested subvolumes; borgmatic selects the subvolume that's the "closest" parent to your source directories. New in version 1.9.6 When using [patterns](https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-help-patterns), the initial portion of a pattern's path that you intend borgmatic to match -against a subvolume can't have globs or other non-literal characters in it—or it -won't actually match. For instance, a subvolume of `/var` would match a pattern -of `+ fm:/var/*/data`, but borgmatic isn't currently smart enough to match -`/var` to a pattern like `+ fm:/v*/lib/data`. +against a subvolume mount point can't have globs or other non-literal characters +in it—or it won't actually match. For instance, a subvolume mount point of +`/var` would match a pattern of `+ fm:/var/*/data`, but borgmatic isn't +currently smart enough to match `/var` to a pattern like `+ fm:/v*/lib/data`. -Additionally, borgmatic rewrites the snapshot file paths so that they appear -at their original subvolume locations in a Borg archive. For instance, if your -subvolume exists at `/var/subvolume`, then the snapshotted files will appear +Additionally, borgmatic rewrites the snapshot file paths so that they appear at +their original subvolume locations in a Borg archive. For instance, if your +subvolume is mounted at `/var/subvolume`, then the snapshotted files will appear in an archive at `/var/subvolume` as well—even if borgmatic has to mount the -snapshot somewhere in `/var/subvolume/.borgmatic-snapshot-1234/` to perform -the backup. +snapshot somewhere in `/var/subvolume/.borgmatic-snapshot-1234/` to perform the +backup. With Borg version 1.2 and earlierSnapshotted files are instead stored at a path dependent on the From 7a766c717e7a6237514d1016d4dd3706766d8e1f Mon Sep 17 00:00:00 2001 From: Gautam Aggarwal Date: Thu, 27 Mar 2025 02:55:16 +0000 Subject: [PATCH 175/226] 2nd Draft --- borgmatic/commands/borgmatic.py | 20 ++++++++++---------- borgmatic/config/schema.yaml | 8 +++++--- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index d8c27034..0a6a775c 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -943,19 +943,18 @@ def exit_with_help_link(): # pragma: no cover sys.exit(1) -def check_and_show_help_on_no_args(): +def check_and_show_help_on_no_args(configs): """ Check if the 'borgmatic' command is run without any arguments. If the configuration option - 'show_help_on_no_args' is set to True, show the help message. Otherwise, trigger the + 'default_actions' is set to False, show the help message. Otherwise, trigger the default backup behavior. """ if len(sys.argv) == 1: # No arguments provided - show_help_on_no_args = any( - config.get('show_help_on_no_args', False) - for config in load_configurations(tuple(collect.collect_config_filenames(None)))[0].values() + default_actions = any( + config.get('default_actions', True) for config in configs.values() ) - if show_help_on_no_args: - print(parse_arguments('--help')) + if not default_actions: + parse_arguments('--help') sys.exit(0) @@ -987,9 +986,6 @@ def main(extra_summary_logs=[]): # pragma: no cover print(borgmatic.commands.completion.fish.fish_completion()) sys.exit(0) - # Use the helper function to check and show help on no arguments - check_and_show_help_on_no_args() - validate = bool('validate' in arguments) config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths)) configs, config_paths, parse_logs = load_configurations( @@ -997,6 +993,10 @@ def main(extra_summary_logs=[]): # pragma: no cover global_arguments.overrides, resolve_env=global_arguments.resolve_env and not validate, ) + + # Use the helper function to check and show help on no arguments, passing the preloaded configs + check_and_show_help_on_no_args(configs) + configuration_parse_errors = ( (max(log.levelno for log in parse_logs) >= logging.CRITICAL) if parse_logs else False ) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 96bba3b9..65ba9f77 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -2665,9 +2665,11 @@ properties: example: /usr/local/bin/keepassxc-cli description: | Configuration for integration with the KeePassXC password manager. - show_help_on_no_args: + default_actions: type: boolean description: | - If true, running borgmatic without any arguments will display the - help message instead of triggering a backup. Defaults to false. + Whether to apply default actions (e.g., backup) when no arguments + are supplied to the borgmatic command. If set to true, borgmatic + will trigger the default actions. If set to false, borgmatic will + display the help message instead. example: true From 486bec698d44b2dd560983f990e8c595683a85a7 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 26 Mar 2025 22:13:30 -0700 Subject: [PATCH 176/226] Add "key import" to reference documentation (#345). --- docs/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Dockerfile b/docs/Dockerfile index c3279570..afa3457d 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -4,7 +4,7 @@ COPY . /app RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib RUN pip install --break-system-packages --no-cache /app && borgmatic config generate && chmod +r /etc/borgmatic/config.yaml RUN borgmatic --help > /command-line.txt \ - && for action in repo-create transfer create prune compact check delete extract config "config bootstrap" "config generate" "config validate" export-tar mount umount repo-delete restore repo-list list repo-info info break-lock "key export" "key change-passphrase" borg; do \ + && for action in repo-create transfer create prune compact check delete extract config "config bootstrap" "config generate" "config validate" export-tar mount umount repo-delete restore repo-list list repo-info info break-lock "key export" "key import" "key change-passphrase" borg; do \ echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \ && borgmatic $action --help >> /command-line.txt; done RUN /app/docs/fetch-contributors >> /contributors.html From 088da190129786a0a092be11086ee43326ff4601 Mon Sep 17 00:00:00 2001 From: Gautam Aggarwal Date: Thu, 27 Mar 2025 11:26:56 +0000 Subject: [PATCH 177/226] Added Unit Tests --- borgmatic/config/schema.yaml | 8 ++++++-- tests/unit/commands/test_borgmatic.py | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 65ba9f77..7a4c4a28 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -2670,6 +2670,10 @@ properties: description: | Whether to apply default actions (e.g., backup) when no arguments are supplied to the borgmatic command. If set to true, borgmatic - will trigger the default actions. If set to false, borgmatic will - display the help message instead. + will trigger the default actions,which include : + -Create backups based on your configured schedules and paths. + - Prune old backups based on your retention policies. + - Verify the integrity of your backups. + + If set to false, borgmatic will display the help message instead. example: true diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 160f4454..15512a67 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -2079,3 +2079,23 @@ def test_collect_configuration_run_summary_logs_outputs_merged_json_results(): arguments=arguments, ) ) +def test_check_and_show_help_on_no_args_shows_help_when_no_args_and_default_actions_false(): + flexmock(module.sys).should_receive('argv').and_return(['borgmatic']) + flexmock(module).should_receive('parse_arguments').with_args('--help').once() + flexmock(module.sys).should_receive('exit').with_args(0).once() + module.check_and_show_help_on_no_args({'test.yaml': {'default_actions': False}}) + + +def test_check_and_show_help_on_no_args_does_not_show_help_when_no_args_and_default_actions_true(): + flexmock(module.sys).should_receive('argv').and_return(['borgmatic']) + flexmock(module).should_receive('parse_arguments').never() + flexmock(module.sys).should_receive('exit').never() + module.check_and_show_help_on_no_args({'test.yaml': {'default_actions': True}}) + + +def test_check_and_show_help_on_no_args_does_not_show_help_when_args_provided(): + flexmock(module.sys).should_receive('argv').and_return(['borgmatic', '--create']) + flexmock(module).should_receive('parse_arguments').never() + flexmock(module.sys).should_receive('exit').never() + module.check_and_show_help_on_no_args({'test.yaml': {'default_actions': False}}) + From 26fd41da92e262e68fc87e24fd1e828937984f50 Mon Sep 17 00:00:00 2001 From: Vandal <86826719+VandalByte@users.noreply.github.com> Date: Thu, 27 Mar 2025 22:18:34 +0530 Subject: [PATCH 178/226] add rest of flags --- borgmatic/actions/recreate.py | 4 ++-- borgmatic/borg/recreate.py | 30 +++++++++++++++--------------- borgmatic/commands/arguments.py | 21 +++++++++++++-------- 3 files changed, 30 insertions(+), 25 deletions(-) diff --git a/borgmatic/actions/recreate.py b/borgmatic/actions/recreate.py index 53ee7fce..3cef7762 100644 --- a/borgmatic/actions/recreate.py +++ b/borgmatic/actions/recreate.py @@ -23,9 +23,9 @@ def run_recreate( repository, recreate_arguments.repository ): if recreate_arguments.archive: - logger.info(f'Recreating archive {recreate_arguments.archive}') + logger.answer(f'Recreating archive {recreate_arguments.archive}') else: - logger.info('Recreating repository') + logger.answer('Recreating repository') # Collect and process patterns. processed_patterns = process_patterns( diff --git a/borgmatic/borg/recreate.py b/borgmatic/borg/recreate.py index 640bcd05..ea2a2081 100644 --- a/borgmatic/borg/recreate.py +++ b/borgmatic/borg/recreate.py @@ -1,4 +1,5 @@ import logging +import shlex from datetime import datetime import borgmatic.borg.environment @@ -10,15 +11,6 @@ from borgmatic.borg.create import make_exclude_flags, make_list_filter_flags, wr logger = logging.getLogger(__name__) -def is_valid_timestamp(time_str): - try: - # Attempt to parse the time string using the expected format - datetime.strptime(time_str, r"%Y-%m-%dT%H:%M:%S") - return True - except ValueError: - return False - - def recreate_archive( repository, archive, @@ -37,15 +29,11 @@ def recreate_archive( Executes the recreate command with the given arguments. ''' - if recreate_arguments.timestamp and not is_valid_timestamp(recreate_arguments.timestamp): - logger.error( - 'Please provide a valid timestamp of format: yyyy-mm-ddThh:mm:ss. Example: 2025-03-26T14:45:59' - ) - return lock_wait = config.get('lock_wait', None) exclude_flags = make_exclude_flags(config) compression = config.get('compression', None) + chunker_params = config.get('chunker_params', None) # Write patterns to a temporary file and use that file with --patterns-from. patterns_file = write_patterns_file( @@ -80,8 +68,20 @@ def recreate_archive( if recreate_arguments.target and recreate_arguments.archive else () ) - + (('--comment', f'"{recreate_arguments.comment}"') if recreate_arguments.comment else ()) + + ( + ('--comment', shlex.quote(recreate_arguments.comment)) + if recreate_arguments.comment + else () + ) + + (('--timestamp', recreate_arguments.timestamp) if recreate_arguments.timestamp else ()) + (('--compression', compression) if compression else ()) + + (('--chunker-params', chunker_params) if chunker_params else ()) + + flags.make_match_archives_flags( + recreate_arguments.match_archives or archive or config.get('match_archives'), + config.get('archive_name_format'), + local_borg_version, + ) + + (('--recompress', recreate_arguments.recompress) if recreate_arguments.recompress else ()) + exclude_flags + ( flags.make_repository_archive_flags(repository, archive, local_borg_version) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index f9d0e7af..689959ce 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1582,17 +1582,22 @@ def make_parsers(): metavar='COMMENT', help='Add a comment text to the archive, if archive not provided, consider all archives', ) - recreate_group.add_argument( - '--compression', - '-C', - dest='compression', - metavar='COMPRESSION', - help='Select the compression algorithm', - ) recreate_group.add_argument( '--timestamp', metavar='TIMESTAMP', - help='Manually specify the archive creation date/time (UTC, yyyy-mm-ddThh:mm:ss format)', + help='Manually specify the archive creation date/time (UTC)', + ) + recreate_group.add_argument( + '-a', + '--match-archives', + '--glob-archives', + metavar='PATTERN', + help='Only consider archive names, hashes, or series matching this pattern', + ) + recreate_group.add_argument( + '--recompress', + metavar='MODE', + help='Recompress data chunks according to MODE: [if-different (default), always, never]', ) recreate_group.add_argument( '-h', '--help', action='help', help='Show this help message and exit' From 50beb334dcd561f9bb7c446ebde80136ef21426f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 27 Mar 2025 11:07:25 -0700 Subject: [PATCH 179/226] Add tests for adding array element arguments and fix the code under test (#303). --- borgmatic/commands/arguments.py | 58 ++++-- tests/integration/commands/test_arguments.py | 36 ++++ tests/unit/actions/test_repo_create.py | 54 ++++- tests/unit/commands/test_arguments.py | 206 +++++++++++++++++++ 4 files changed, 331 insertions(+), 23 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 97dc3361..8a92efb9 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -299,31 +299,32 @@ def make_argument_description(schema, flag_name): ''' description = schema.get('description') schema_type = schema.get('type') + example = schema.get('example') if not description: return None - if schema_type == 'array': + if '[0]' in flag_name: + description += ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).' + + if example and schema_type == 'array': example_buffer = io.StringIO() yaml = ruamel.yaml.YAML(typ='safe') yaml.default_flow_style = True - yaml.dump(schema.get('example'), example_buffer) + yaml.dump(example, example_buffer) description += f' Example value: "{example_buffer.getvalue().strip()}"' - if '[0]' in flag_name: - description += ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).' - description = description.replace('%', '%%') return description -def add_array_element_arguments_from_schema(arguments_group, schema, unparsed_arguments, flag_name): +def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): r''' - Given an argparse._ArgumentGroup instance, a configuration schema dict, a sequence of unparsed - argument strings, and a dotted flag name, convert the schema into corresponding command-line - array element flags that correspond to the given unparsed arguments. + Given an argparse._ArgumentGroup instance, a sequence of unparsed argument strings, and a dotted + flag name, convert the schema into corresponding command-line array element flags that + correspond to the given unparsed arguments. Here's the background. We want to support flags that can have arbitrary indices like: @@ -356,26 +357,39 @@ def add_array_element_arguments_from_schema(arguments_group, schema, unparsed_ar argument will match it when parsing is performed! In this manner, we're using the actual user CLI input to inform what exact flags we support! ''' - if '[0]' not in flag_name or '--help' in unparsed_arguments: + if '[0]' not in flag_name or not unparsed_arguments or '--help' in unparsed_arguments: return pattern = re.compile(fr'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.")}$') - existing_flags = set( - itertools.chain( - *(group_action.option_strings for group_action in arguments_group._group_actions) + + # Find an existing list index flag (and its action) corresponding to the given flag name. If one + # isn't found, bail. + try: + (argument_action, existing_flag_name) = next( + (action, action_flag_name) + for action in arguments_group._group_actions + for action_flag_name in action.option_strings + if pattern.match(action_flag_name) + if f'--{flag_name}'.startswith(action_flag_name) ) - ) + except StopIteration: + return for unparsed in unparsed_arguments: unparsed_flag_name = unparsed.split('=', 1)[0] - if pattern.match(unparsed_flag_name) and unparsed_flag_name not in existing_flags: - arguments_group.add_argument( - unparsed_flag_name, - type=argument_type, - metavar=metavar, - help=description, - ) + if not pattern.match(unparsed_flag_name) or unparsed_flag_name == existing_flag_name: + continue + + arguments_group.add_argument( + unparsed_flag_name, + choices=argument_action.choices, + default=argument_action.default, + dest=unparsed_flag_name.lstrip('-'), + nargs=argument_action.nargs, + required=argument_action.nargs, + type=argument_action.type, + ) def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names=None): @@ -482,7 +496,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names help=description, ) - add_array_element_arguments_from_schema(arguments_group, schema, unparsed_arguments, flag_name) + add_array_element_arguments(arguments_group, unparsed_arguments, flag_name) def make_parsers(schema, unparsed_arguments): diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 3469858c..5f539660 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -4,6 +4,42 @@ from flexmock import flexmock from borgmatic.commands import arguments as module +def test_make_argument_description_with_array_adds_example(): + assert module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'array', + 'example': [1, '- foo', {'bar': 'baz'}], + }, + flag_name='flag', + ) == 'Thing. Example value: "[1, \'- foo\', bar: baz]"' + + +def test_add_array_element_arguments_adds_arguments_for_array_index_flags(): + parser = module.ArgumentParser(allow_abbrev=False, add_help=False) + arguments_group = parser.add_argument_group('arguments') + arguments_group.add_argument( + '--foo[0].val', + dest='--foo[0].val', + ) + + flexmock(arguments_group).should_receive('add_argument').with_args( + '--foo[25].val', + choices=object, + default=object, + dest='foo[25].val', + nargs=object, + required=object, + type=object, + ).once() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo[25].val', 'fooval', '--bar[1].val', 'barval'), + flag_name='foo[0].val', + ) + + def test_parse_arguments_with_no_arguments_uses_defaults(): config_paths = ['default'] flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) diff --git a/tests/unit/actions/test_repo_create.py b/tests/unit/actions/test_repo_create.py index 8bb350b1..035dd623 100644 --- a/tests/unit/actions/test_repo_create.py +++ b/tests/unit/actions/test_repo_create.py @@ -1,9 +1,10 @@ +import pytest from flexmock import flexmock from borgmatic.actions import repo_create as module -def test_run_repo_create_does_not_raise(): +def test_run_repo_create_with_encryption_mode_argument_does_not_raise(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) flexmock(module.borgmatic.borg.repo_create).should_receive('create_repository') @@ -28,6 +29,57 @@ def test_run_repo_create_does_not_raise(): ) +def test_run_repo_create_with_encryption_mode_option_does_not_raise(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.repo_create).should_receive('create_repository') + arguments = flexmock( + encryption_mode=None, + source_repository=flexmock(), + repository=flexmock(), + copy_crypt_key=flexmock(), + append_only=flexmock(), + storage_quota=flexmock(), + make_parent_dirs=flexmock(), + ) + + module.run_repo_create( + repository={'path': 'repo', 'encryption': flexmock()}, + config={}, + local_borg_version=None, + repo_create_arguments=arguments, + global_arguments=flexmock(dry_run=False), + local_path=None, + remote_path=None, + ) + + +def test_run_repo_create_without_encryption_mode_raises(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.repo_create).should_receive('create_repository') + arguments = flexmock( + encryption_mode=None, + source_repository=flexmock(), + repository=flexmock(), + copy_crypt_key=flexmock(), + append_only=flexmock(), + storage_quota=flexmock(), + make_parent_dirs=flexmock(), + ) + + with pytest.raises(ValueError): + module.run_repo_create( + repository={'path': 'repo'}, + config={}, + local_borg_version=None, + repo_create_arguments=arguments, + global_arguments=flexmock(dry_run=False), + local_path=None, + remote_path=None, + ) + + def test_run_repo_create_bails_if_repository_does_not_match(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return( diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index d10cf887..379dba3b 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -575,3 +575,209 @@ def test_parse_arguments_for_actions_raises_error_when_no_action_is_specified(): with pytest.raises(ValueError): module.parse_arguments_for_actions(('config',), action_parsers, global_parser) + + +def test_make_argument_description_without_description_bails(): + assert module.make_argument_description( + schema={ + 'description': None, + 'type': 'not yours', + }, + flag_name='flag', + ) is None + + +def test_make_argument_description_with_array_adds_example(): + buffer = flexmock() + buffer.should_receive('getvalue').and_return('[example]') + flexmock(module.io).should_receive('StringIO').and_return(buffer) + yaml = flexmock() + yaml.should_receive('dump') + flexmock(module.ruamel.yaml).should_receive('YAML').and_return(yaml) + + assert module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'array', + 'example': ['example'], + }, + flag_name='flag', + ) == 'Thing. Example value: "[example]"' + + +def test_make_argument_description_with_array_skips_missing_example(): + yaml = flexmock() + yaml.should_receive('dump').and_return('[example]') + flexmock(module.ruamel.yaml).should_receive('YAML').and_return(yaml) + + assert module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'array', + }, + flag_name='flag', + ) == 'Thing.' + + +def test_make_argument_description_with_array_index_in_flag_name_adds_to_description(): + assert 'list element' in module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'something', + }, + flag_name='flag[0]', + ) + + +def test_make_argument_description_escapes_percent_character(): + assert module.make_argument_description( + schema={ + 'description': '% Thing.', + 'type': 'something', + }, + flag_name='flag', + ) == '%% Thing.' + + +def test_add_array_element_arguments_without_array_index_bails(): + arguments_group = flexmock() + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=(), + flag_name='foo', + ) + + +def test_add_array_element_arguments_with_help_flag_bails(): + arguments_group = flexmock() + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo', '--help', '--bar'), + flag_name='foo[0]', + ) + + +def test_add_array_element_arguments_without_any_flags_bails(): + arguments_group = flexmock() + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=(), + flag_name='foo[0]', + ) + + +def test_add_array_element_arguments_without_array_index_flags_bails(): + arguments_group = flexmock( + _group_actions=( + flexmock( + option_strings=('--foo[0].val',), + ), + ) + ) + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo', '--bar'), + flag_name='foo[0].val', + ) + + +def test_add_array_element_arguments_with_non_matching_array_index_flags_bails(): + arguments_group = flexmock( + _group_actions=( + flexmock( + option_strings=('--foo[0].val',), + ), + ) + ) + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo', '--bar[25].val', 'barval'), + flag_name='foo[0].val', + ) + + +def test_add_array_element_arguments_with_identical_array_index_flag_bails(): + arguments_group = flexmock( + _group_actions=( + flexmock( + option_strings=('--foo[0].val',), + ), + ) + ) + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo[0].val', 'fooval', '--bar'), + flag_name='foo[0].val', + ) + + +def test_add_array_element_arguments_adds_arguments_for_array_index_flags(): + arguments_group = flexmock( + _group_actions=( + flexmock( + option_strings=('--foo[0].val',), + choices=flexmock(), + default=flexmock(), + nargs=flexmock(), + required=flexmock(), + type=flexmock(), + ), + ) + ) + arguments_group.should_receive('add_argument').with_args( + '--foo[25].val', + choices=object, + default=object, + dest='foo[25].val', + nargs=object, + required=object, + type=object, + ).once() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo[25].val', 'fooval', '--bar[1].val', 'barval'), + flag_name='foo[0].val', + ) + + +def test_add_array_element_arguments_adds_arguments_for_array_index_flags_with_equals_sign(): + arguments_group = flexmock( + _group_actions=( + flexmock( + option_strings=('--foo[0].val',), + choices=flexmock(), + default=flexmock(), + nargs=flexmock(), + required=flexmock(), + type=flexmock(), + ), + ) + ) + arguments_group.should_receive('add_argument').with_args( + '--foo[25].val', + choices=object, + default=object, + dest='foo[25].val', + nargs=object, + required=object, + type=object, + ).once() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo[25].val=fooval', '--bar[1].val=barval'), + flag_name='foo[0].val', + ) From 79bf641668bcf4ea438d670c8e7cebc28ab75086 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 27 Mar 2025 12:42:49 -0700 Subject: [PATCH 180/226] Set the action type when cloning an argument for a list index flag (#303). --- borgmatic/commands/arguments.py | 13 +++- tests/integration/commands/test_arguments.py | 2 + tests/unit/commands/test_arguments.py | 69 ++++++++++++++++---- 3 files changed, 71 insertions(+), 13 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 8a92efb9..8e0a7d85 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -362,9 +362,8 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): pattern = re.compile(fr'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.")}$') - # Find an existing list index flag (and its action) corresponding to the given flag name. If one - # isn't found, bail. try: + # Find an existing list index flag (and its action) corresponding to the given flag name. (argument_action, existing_flag_name) = next( (action, action_flag_name) for action in arguments_group._group_actions @@ -372,6 +371,15 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): if pattern.match(action_flag_name) if f'--{flag_name}'.startswith(action_flag_name) ) + + # Based on the type of the action (e.g. argparse._StoreTrueAction), look up the corresponding + # action registry name (e.g., "store_true") to pass to add_argument(action=...) below. + action_registry_name = next( + registry_name + for registry_name, action_type in arguments_group._registries['action'].items() + # Not using isinstance() here because we only want an exact match—no parent classes. + if type(argument_action) == action_type + ) except StopIteration: return @@ -383,6 +391,7 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): arguments_group.add_argument( unparsed_flag_name, + action=action_registry_name, choices=argument_action.choices, default=argument_action.default, dest=unparsed_flag_name.lstrip('-'), diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 5f539660..d6841a2c 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -20,11 +20,13 @@ def test_add_array_element_arguments_adds_arguments_for_array_index_flags(): arguments_group = parser.add_argument_group('arguments') arguments_group.add_argument( '--foo[0].val', + action='store_true', dest='--foo[0].val', ) flexmock(arguments_group).should_receive('add_argument').with_args( '--foo[25].val', + action='store_true', choices=object, default=object, dest='foo[25].val', diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index 379dba3b..a9479db5 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -672,13 +672,31 @@ def test_add_array_element_arguments_without_any_flags_bails(): ) +# Use this instead of a flexmock because it's not easy to check the type() of a flexmock instance. +Group_action = collections.namedtuple( + 'Group_action', + ( + 'option_strings', + 'choices', + 'default', + 'nargs', + 'required', + 'type', + ), + defaults=( + flexmock(), flexmock(), flexmock(), flexmock(), flexmock(), + ) +) + + def test_add_array_element_arguments_without_array_index_flags_bails(): arguments_group = flexmock( _group_actions=( - flexmock( + Group_action( option_strings=('--foo[0].val',), ), - ) + ), + _registries={'action': {'store_stuff': Group_action}}, ) arguments_group.should_receive('add_argument').never() @@ -692,10 +710,11 @@ def test_add_array_element_arguments_without_array_index_flags_bails(): def test_add_array_element_arguments_with_non_matching_array_index_flags_bails(): arguments_group = flexmock( _group_actions=( - flexmock( + Group_action( option_strings=('--foo[0].val',), ), - ) + ), + _registries={'action': {'store_stuff': Group_action}}, ) arguments_group.should_receive('add_argument').never() @@ -709,10 +728,11 @@ def test_add_array_element_arguments_with_non_matching_array_index_flags_bails() def test_add_array_element_arguments_with_identical_array_index_flag_bails(): arguments_group = flexmock( _group_actions=( - flexmock( + Group_action( option_strings=('--foo[0].val',), ), - ) + ), + _registries={'action': {'store_stuff': Group_action}}, ) arguments_group.should_receive('add_argument').never() @@ -723,10 +743,10 @@ def test_add_array_element_arguments_with_identical_array_index_flag_bails(): ) -def test_add_array_element_arguments_adds_arguments_for_array_index_flags(): +def test_add_array_element_arguments_without_action_type_in_registry_bails(): arguments_group = flexmock( _group_actions=( - flexmock( + Group_action( option_strings=('--foo[0].val',), choices=flexmock(), default=flexmock(), @@ -734,10 +754,35 @@ def test_add_array_element_arguments_adds_arguments_for_array_index_flags(): required=flexmock(), type=flexmock(), ), - ) + ), + _registries={'action': {'store_stuff': bool}}, + ) + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo[25].val', 'fooval', '--bar[1].val', 'barval'), + flag_name='foo[0].val', + ) + + +def test_add_array_element_arguments_adds_arguments_for_array_index_flags(): + arguments_group = flexmock( + _group_actions=( + Group_action( + option_strings=('--foo[0].val',), + choices=flexmock(), + default=flexmock(), + nargs=flexmock(), + required=flexmock(), + type=flexmock(), + ), + ), + _registries={'action': {'store_stuff': Group_action}}, ) arguments_group.should_receive('add_argument').with_args( '--foo[25].val', + action='store_stuff', choices=object, default=object, dest='foo[25].val', @@ -756,7 +801,7 @@ def test_add_array_element_arguments_adds_arguments_for_array_index_flags(): def test_add_array_element_arguments_adds_arguments_for_array_index_flags_with_equals_sign(): arguments_group = flexmock( _group_actions=( - flexmock( + Group_action( option_strings=('--foo[0].val',), choices=flexmock(), default=flexmock(), @@ -764,10 +809,12 @@ def test_add_array_element_arguments_adds_arguments_for_array_index_flags_with_e required=flexmock(), type=flexmock(), ), - ) + ), + _registries={'action': {'store_stuff': Group_action}}, ) arguments_group.should_receive('add_argument').with_args( '--foo[25].val', + action='store_stuff', choices=object, default=object, dest='foo[25].val', From b4c558d013b4c322dcc6ef8d447b7763cc1f1735 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 27 Mar 2025 16:49:14 -0700 Subject: [PATCH 181/226] Add tests for CLI arguments from schema logic (#303). --- borgmatic/commands/arguments.py | 11 +- borgmatic/config/schema.py | 2 + tests/integration/commands/test_arguments.py | 93 ++++- tests/unit/commands/test_arguments.py | 415 +++++++++++++++++-- 4 files changed, 476 insertions(+), 45 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 8e0a7d85..3bd22082 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -323,8 +323,7 @@ def make_argument_description(schema, flag_name): def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): r''' Given an argparse._ArgumentGroup instance, a sequence of unparsed argument strings, and a dotted - flag name, convert the schema into corresponding command-line array element flags that - correspond to the given unparsed arguments. + flag name, add command-line array element flags that correspond to the given unparsed arguments. Here's the background. We want to support flags that can have arbitrary indices like: @@ -353,9 +352,9 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): --foo.bar[1].baz - ... then an argument flag will get added equal to that unparsed argument. And the unparsed + ... then an argument flag will get added equal to that unparsed argument. And so the unparsed argument will match it when parsing is performed! In this manner, we're using the actual user - CLI input to inform what exact flags we support! + CLI input to inform what exact flags we support. ''' if '[0]' not in flag_name or not unparsed_arguments or '--help' in unparsed_arguments: return @@ -462,8 +461,8 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names return - # If this is an "array" type, recurse for each child option of its items type. Don't return yet, - # so that a flag also gets added below for the array itself. + # If this is an "array" type, recurse for each items type child option. Don't return yet so that + # a flag also gets added below for the array itself. if schema_type == 'array': properties = borgmatic.config.schema.get_properties(schema.get('items', {})) diff --git a/borgmatic/config/schema.py b/borgmatic/config/schema.py index 0ce774d9..52358a3e 100644 --- a/borgmatic/config/schema.py +++ b/borgmatic/config/schema.py @@ -35,6 +35,8 @@ def parse_type(schema_type): 'integer': int, 'number': decimal.Decimal, 'boolean': bool, + # This is str instead of list to support specifying a list as a YAML string on the + # command-line. 'array': str, }[schema_type] except KeyError: diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index d6841a2c..200368aa 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -5,14 +5,17 @@ from borgmatic.commands import arguments as module def test_make_argument_description_with_array_adds_example(): - assert module.make_argument_description( - schema={ - 'description': 'Thing.', - 'type': 'array', - 'example': [1, '- foo', {'bar': 'baz'}], - }, - flag_name='flag', - ) == 'Thing. Example value: "[1, \'- foo\', bar: baz]"' + assert ( + module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'array', + 'example': [1, '- foo', {'bar': 'baz'}], + }, + flag_name='flag', + ) + == 'Thing. Example value: "[1, \'- foo\', bar: baz]"' + ) def test_add_array_element_arguments_adds_arguments_for_array_index_flags(): @@ -42,6 +45,80 @@ def test_add_array_element_arguments_adds_arguments_for_array_index_flags(): ) +def test_add_arguments_from_schema_with_nested_object_adds_flag_for_each_option(): + parser = module.ArgumentParser(allow_abbrev=False, add_help=False) + arguments_group = parser.add_argument_group('arguments') + flexmock(arguments_group).should_receive('add_argument').with_args( + '--foo.bar', + type=int, + metavar='BAR', + help='help 1', + ).once() + flexmock(arguments_group).should_receive('add_argument').with_args( + '--foo.baz', + type=str, + metavar='BAZ', + help='help 2', + ).once() + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'object', + 'properties': { + 'bar': {'type': 'integer', 'description': 'help 1'}, + 'baz': {'type': 'string', 'description': 'help 2'}, + } + } + } + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_array_and_nested_object_adds_multiple_flags(): + parser = module.ArgumentParser(allow_abbrev=False, add_help=False) + arguments_group = parser.add_argument_group('arguments') + flexmock(arguments_group).should_receive('add_argument').with_args( + '--foo[0].bar', + type=int, + metavar='BAR', + help=object, + ).once() + flexmock(arguments_group).should_receive('add_argument').with_args( + '--foo', + type=str, + metavar='FOO', + help='help 2', + ).once() + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'bar': { + 'type': 'integer', + 'description': 'help 1', + } + } + }, + 'description': 'help 2', + } + } + }, + unparsed_arguments=(), + ) + + def test_parse_arguments_with_no_arguments_uses_defaults(): config_paths = ['default'] flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index a9479db5..fcfa9589 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -578,13 +578,16 @@ def test_parse_arguments_for_actions_raises_error_when_no_action_is_specified(): def test_make_argument_description_without_description_bails(): - assert module.make_argument_description( - schema={ - 'description': None, - 'type': 'not yours', - }, - flag_name='flag', - ) is None + assert ( + module.make_argument_description( + schema={ + 'description': None, + 'type': 'not yours', + }, + flag_name='flag', + ) + is None + ) def test_make_argument_description_with_array_adds_example(): @@ -595,14 +598,17 @@ def test_make_argument_description_with_array_adds_example(): yaml.should_receive('dump') flexmock(module.ruamel.yaml).should_receive('YAML').and_return(yaml) - assert module.make_argument_description( - schema={ - 'description': 'Thing.', - 'type': 'array', - 'example': ['example'], - }, - flag_name='flag', - ) == 'Thing. Example value: "[example]"' + assert ( + module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'array', + 'example': ['example'], + }, + flag_name='flag', + ) + == 'Thing. Example value: "[example]"' + ) def test_make_argument_description_with_array_skips_missing_example(): @@ -610,13 +616,16 @@ def test_make_argument_description_with_array_skips_missing_example(): yaml.should_receive('dump').and_return('[example]') flexmock(module.ruamel.yaml).should_receive('YAML').and_return(yaml) - assert module.make_argument_description( - schema={ - 'description': 'Thing.', - 'type': 'array', - }, - flag_name='flag', - ) == 'Thing.' + assert ( + module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'array', + }, + flag_name='flag', + ) + == 'Thing.' + ) def test_make_argument_description_with_array_index_in_flag_name_adds_to_description(): @@ -630,13 +639,16 @@ def test_make_argument_description_with_array_index_in_flag_name_adds_to_descrip def test_make_argument_description_escapes_percent_character(): - assert module.make_argument_description( - schema={ - 'description': '% Thing.', - 'type': 'something', - }, - flag_name='flag', - ) == '%% Thing.' + assert ( + module.make_argument_description( + schema={ + 'description': '% Thing.', + 'type': 'something', + }, + flag_name='flag', + ) + == '%% Thing.' + ) def test_add_array_element_arguments_without_array_index_bails(): @@ -684,8 +696,12 @@ Group_action = collections.namedtuple( 'type', ), defaults=( - flexmock(), flexmock(), flexmock(), flexmock(), flexmock(), - ) + flexmock(), + flexmock(), + flexmock(), + flexmock(), + flexmock(), + ), ) @@ -828,3 +844,340 @@ def test_add_array_element_arguments_adds_arguments_for_array_index_flags_with_e unparsed_arguments=('--foo[25].val=fooval', '--bar[1].val=barval'), flag_name='foo[0].val', ) + + +def test_add_arguments_from_schema_with_non_dict_schema_bails(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').never() + flexmock(module.borgmatic.config.schema).should_receive('parse_type').never() + arguments_group.should_receive('add_argument').never() + + module.add_arguments_from_schema( + arguments_group=arguments_group, schema='foo', unparsed_arguments=() + ) + + +def test_add_arguments_from_schema_with_nested_object_adds_flag_for_each_option(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help 1').and_return('help 2') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(int).and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--foo.bar', + type=int, + metavar='BAR', + help='help 1', + ).once() + arguments_group.should_receive('add_argument').with_args( + '--foo.baz', + type=str, + metavar='BAZ', + help='help 2', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'object', + 'properties': { + 'bar': {'type': 'integer'}, + 'baz': {'type': 'str'}, + } + } + } + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_uses_first_non_null_type_from_multi_type_object(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help 1') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(int) + arguments_group.should_receive('add_argument').with_args( + '--foo.bar', + type=int, + metavar='BAR', + help='help 1', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': ['null', 'object', 'boolean'], + 'properties': { + 'bar': {'type': 'integer'}, + } + } + } + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_empty_multi_type_raises(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help 1') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(int) + arguments_group.should_receive('add_argument').never() + flexmock(module).should_receive('add_array_element_arguments').never() + + with pytest.raises(ValueError): + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': [], + 'properties': { + 'bar': {'type': 'integer'}, + } + } + } + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_propertyless_option_does_not_add_flag(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').never() + flexmock(module.borgmatic.config.schema).should_receive('parse_type').never() + arguments_group.should_receive('add_argument').never() + flexmock(module).should_receive('add_array_element_arguments').never() + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'object', + } + } + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_array_adds_flag(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--foo', + type=str, + metavar='FOO', + help='help', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'array', + 'items': { + 'type': 'integer', + } + } + } + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_array_and_nested_object_adds_multiple_flags(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help 1').and_return('help 2') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(int).and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--foo[0].bar', + type=int, + metavar='BAR', + help='help 1', + ).once() + arguments_group.should_receive('add_argument').with_args( + '--foo', + type=str, + metavar='FOO', + help='help 2', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + flexmock(module).should_receive('add_array_element_arguments').with_args( + arguments_group=arguments_group, + unparsed_arguments=(), + flag_name='foo[0].bar', + ).once() + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'bar': { + 'type': 'integer', + } + } + } + } + } + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_default_false_boolean_adds_valueless_flag(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(bool) + arguments_group.should_receive('add_argument').with_args( + '--foo', + action='store_true', + default=None, + help='help', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'boolean', + 'default': False, + } + } + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_default_true_boolean_adds_value_flag(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(bool) + arguments_group.should_receive('add_argument').with_args( + '--foo', + type=bool, + metavar='FOO', + help='help', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'boolean', + 'default': True, + } + } + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_defaultless_boolean_adds_value_flag(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(bool) + arguments_group.should_receive('add_argument').with_args( + '--foo', + type=bool, + metavar='FOO', + help='help', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'boolean', + } + } + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_skips_omitted_flag_name(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--match-archives', + type=object, + metavar=object, + help=object, + ).never() + arguments_group.should_receive('add_argument').with_args( + '--foo', + type=str, + metavar='FOO', + help='help', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'match_archives': { + 'type': 'string', + }, + 'foo': { + 'type': 'string', + }, + } + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_rewrites_option_name_to_flag_name(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--foo-and-stuff', + type=str, + metavar='FOO_AND_STUFF', + help='help', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo_and_stuff': { + 'type': 'string', + }, + } + }, + unparsed_arguments=(), + ) From 5bf2f546b9db76ed2863fcb9e982adb15f5983f0 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 27 Mar 2025 21:01:56 -0700 Subject: [PATCH 182/226] More automated tests (#303). --- borgmatic/commands/arguments.py | 5 +- borgmatic/config/arguments.py | 51 ++--- borgmatic/config/schema.py | 25 ++- tests/integration/commands/test_arguments.py | 8 +- tests/integration/config/test_arguments.py | 35 ++++ tests/unit/commands/test_arguments.py | 50 +++-- tests/unit/config/test_arguments.py | 195 +++++++++++++++++++ 7 files changed, 310 insertions(+), 59 deletions(-) create mode 100644 tests/integration/config/test_arguments.py create mode 100644 tests/unit/config/test_arguments.py diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 3bd22082..140dc1ae 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -484,7 +484,10 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names metavar = names[-1].upper() description = make_argument_description(schema, flag_name) - argument_type = borgmatic.config.schema.parse_type(schema_type) + + # array=str instead of list here to support specifying a list as a YAML string on the + # command-line. + argument_type = borgmatic.config.schema.parse_type(schema_type, array=str) full_flag_name = f"--{flag_name.replace('_', '-')}" # As a UX nicety, allow boolean options that have a default of false to have command-line flags diff --git a/borgmatic/config/arguments.py b/borgmatic/config/arguments.py index 8f7023cd..85c8eef9 100644 --- a/borgmatic/config/arguments.py +++ b/borgmatic/config/arguments.py @@ -3,6 +3,8 @@ import re import ruamel.yaml +import borgmatic.config.schema + LIST_INDEX_KEY_PATTERN = re.compile(r'^(?P[a-zA-z-]+)\[(?P\d+)\]$') @@ -45,7 +47,7 @@ def set_values(config, keys, value): try: set_values(config[list_key][list_index], keys[1:], value) except IndexError: - raise ValueError(f'The list index {first_key} is out of range') + raise ValueError(f'Argument list index {first_key} is out of range') return @@ -99,6 +101,7 @@ def convert_value_type(value, option_type): And if the source value isn't a string, return it as-is. Raise ruamel.yaml.error.YAMLError if there's a parse issue with the YAML. + Raise ValueError if the parsed value doesn't match the option type. ''' if not isinstance(value, str): return value @@ -106,7 +109,15 @@ def convert_value_type(value, option_type): if option_type == 'string': return value - return ruamel.yaml.YAML(typ='safe').load(io.StringIO(value)) + try: + parsed_value = ruamel.yaml.YAML(typ='safe').load(io.StringIO(value)) + except ruamel.yaml.error.YAMLError as error: + raise ValueError(f'Argument value "{value}" is invalid: {error.problem}') + + if not isinstance(parsed_value, borgmatic.config.schema.parse_type(option_type)): + raise ValueError(f'Argument value "{value}" is not of the expected type: {option_type}') + + return parsed_value def prepare_arguments_for_config(global_arguments, schema): @@ -122,39 +133,34 @@ def prepare_arguments_for_config(global_arguments, schema): ( (('my_option', 'sub_option'), 'value1'), - (('other_option'), 'value2'), + (('other_option',), 'value2'), ) - - Raise ValueError if an override can't be parsed. ''' prepared_values = [] for argument_name, value in global_arguments.__dict__.items(): - try: - if value is None: - continue + if value is None: + continue - keys = tuple(argument_name.split('.')) - option_type = type_for_option(schema, keys) + keys = tuple(argument_name.split('.')) + option_type = type_for_option(schema, keys) - # The argument doesn't correspond to any option in the schema, so ignore it. It's - # probably a flag that borgmatic has on the command-line but not in configuration. - if option_type is None: - continue + # The argument doesn't correspond to any option in the schema, so ignore it. It's + # probably a flag that borgmatic has on the command-line but not in configuration. + if option_type is None: + continue - prepared_values.append( - ( - keys, - convert_value_type(value, option_type), - ) + prepared_values.append( + ( + keys, + convert_value_type(value, option_type), ) - except ruamel.yaml.error.YAMLError as error: - raise ValueError(f'Invalid override "{argument_name}": {error.problem}') + ) return tuple(prepared_values) -def apply_arguments_to_config(config, schema, global_arguments): +def apply_arguments_to_config(config, schema, global_arguments): # pragma: no cover ''' Given a configuration dict, a corresponding configuration schema dict, and global arguments as an argparse.Namespace, set those given argument values into their corresponding configuration @@ -164,6 +170,5 @@ def apply_arguments_to_config(config, schema, global_arguments): configuration object. Additionally, flags like "--foo.bar[0].baz" are supported to update a list element in the configuration. ''' - for keys, value in prepare_arguments_for_config(global_arguments, schema): set_values(config, keys, value) diff --git a/borgmatic/config/schema.py b/borgmatic/config/schema.py index 52358a3e..6981dde5 100644 --- a/borgmatic/config/schema.py +++ b/borgmatic/config/schema.py @@ -23,22 +23,27 @@ def get_properties(schema): return schema.get('properties', {}) -def parse_type(schema_type): +def parse_type(schema_type, **overrides): ''' Given a schema type as a string, return the corresponding Python type. + If any overrides are given in the from of a schema type string to a Python type, then override + the default type mapping with them. + Raise ValueError if the schema type is unknown. ''' try: - return { - 'string': str, - 'integer': int, - 'number': decimal.Decimal, - 'boolean': bool, - # This is str instead of list to support specifying a list as a YAML string on the - # command-line. - 'array': str, - }[schema_type] + return dict( + { + 'array': list, + 'boolean': bool, + 'integer': int, + 'number': decimal.Decimal, + 'object': dict, + 'string': str, + }, + **overrides, + )[schema_type] except KeyError: raise ValueError(f'Unknown type in configuration schema: {schema_type}') diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 200368aa..032b9495 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -71,9 +71,9 @@ def test_add_arguments_from_schema_with_nested_object_adds_flag_for_each_option( 'properties': { 'bar': {'type': 'integer', 'description': 'help 1'}, 'baz': {'type': 'string', 'description': 'help 2'}, - } + }, } - } + }, }, unparsed_arguments=(), ) @@ -109,11 +109,11 @@ def test_add_arguments_from_schema_with_array_and_nested_object_adds_multiple_fl 'type': 'integer', 'description': 'help 1', } - } + }, }, 'description': 'help 2', } - } + }, }, unparsed_arguments=(), ) diff --git a/tests/integration/config/test_arguments.py b/tests/integration/config/test_arguments.py new file mode 100644 index 00000000..449964ee --- /dev/null +++ b/tests/integration/config/test_arguments.py @@ -0,0 +1,35 @@ +import pytest +from flexmock import flexmock + +from borgmatic.config import arguments as module + + +def test_convert_value_type_passes_through_non_string_value(): + assert module.convert_value_type([1, 2], 'array') == [1, 2] + + +def test_convert_value_type_passes_through_string_option_type(): + assert module.convert_value_type('foo', 'string') == 'foo' + + +def test_convert_value_type_parses_array_option_type(): + assert module.convert_value_type('[foo, bar]', 'array') == ['foo', 'bar'] + + +def test_convert_value_type_with_array_option_type_and_no_array_raises(): + with pytest.raises(ValueError): + module.convert_value_type('{foo, bar}', 'array') + + +def test_convert_value_type_parses_object_option_type(): + assert module.convert_value_type('{foo: bar}', 'object') == {'foo': 'bar'} + + +def test_convert_value_type_with_invalid_value_raises(): + with pytest.raises(ValueError): + module.convert_value_type('{foo, bar', 'object') + + +def test_convert_value_type_with_unknown_option_type_raises(): + with pytest.raises(ValueError): + module.convert_value_type('{foo, bar}', 'thingy') diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index fcfa9589..1e5bbe0b 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -859,8 +859,12 @@ def test_add_arguments_from_schema_with_non_dict_schema_bails(): def test_add_arguments_from_schema_with_nested_object_adds_flag_for_each_option(): arguments_group = flexmock() - flexmock(module).should_receive('make_argument_description').and_return('help 1').and_return('help 2') - flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(int).and_return(str) + flexmock(module).should_receive('make_argument_description').and_return('help 1').and_return( + 'help 2' + ) + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return( + int + ).and_return(str) arguments_group.should_receive('add_argument').with_args( '--foo.bar', type=int, @@ -885,9 +889,9 @@ def test_add_arguments_from_schema_with_nested_object_adds_flag_for_each_option( 'properties': { 'bar': {'type': 'integer'}, 'baz': {'type': 'str'}, - } + }, } - } + }, }, unparsed_arguments=(), ) @@ -914,9 +918,9 @@ def test_add_arguments_from_schema_uses_first_non_null_type_from_multi_type_obje 'type': ['null', 'object', 'boolean'], 'properties': { 'bar': {'type': 'integer'}, - } + }, } - } + }, }, unparsed_arguments=(), ) @@ -939,9 +943,9 @@ def test_add_arguments_from_schema_with_empty_multi_type_raises(): 'type': [], 'properties': { 'bar': {'type': 'integer'}, - } + }, } - } + }, }, unparsed_arguments=(), ) @@ -962,7 +966,7 @@ def test_add_arguments_from_schema_with_propertyless_option_does_not_add_flag(): 'foo': { 'type': 'object', } - } + }, }, unparsed_arguments=(), ) @@ -989,9 +993,9 @@ def test_add_arguments_from_schema_with_array_adds_flag(): 'type': 'array', 'items': { 'type': 'integer', - } + }, } - } + }, }, unparsed_arguments=(), ) @@ -999,8 +1003,12 @@ def test_add_arguments_from_schema_with_array_adds_flag(): def test_add_arguments_from_schema_with_array_and_nested_object_adds_multiple_flags(): arguments_group = flexmock() - flexmock(module).should_receive('make_argument_description').and_return('help 1').and_return('help 2') - flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(int).and_return(str) + flexmock(module).should_receive('make_argument_description').and_return('help 1').and_return( + 'help 2' + ) + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return( + int + ).and_return(str) arguments_group.should_receive('add_argument').with_args( '--foo[0].bar', type=int, @@ -1033,10 +1041,10 @@ def test_add_arguments_from_schema_with_array_and_nested_object_adds_multiple_fl 'bar': { 'type': 'integer', } - } - } + }, + }, } - } + }, }, unparsed_arguments=(), ) @@ -1063,7 +1071,7 @@ def test_add_arguments_from_schema_with_default_false_boolean_adds_valueless_fla 'type': 'boolean', 'default': False, } - } + }, }, unparsed_arguments=(), ) @@ -1090,7 +1098,7 @@ def test_add_arguments_from_schema_with_default_true_boolean_adds_value_flag(): 'type': 'boolean', 'default': True, } - } + }, }, unparsed_arguments=(), ) @@ -1116,7 +1124,7 @@ def test_add_arguments_from_schema_with_defaultless_boolean_adds_value_flag(): 'foo': { 'type': 'boolean', } - } + }, }, unparsed_arguments=(), ) @@ -1151,7 +1159,7 @@ def test_add_arguments_from_schema_skips_omitted_flag_name(): 'foo': { 'type': 'string', }, - } + }, }, unparsed_arguments=(), ) @@ -1177,7 +1185,7 @@ def test_add_arguments_from_schema_rewrites_option_name_to_flag_name(): 'foo_and_stuff': { 'type': 'string', }, - } + }, }, unparsed_arguments=(), ) diff --git a/tests/unit/config/test_arguments.py b/tests/unit/config/test_arguments.py new file mode 100644 index 00000000..25c8873f --- /dev/null +++ b/tests/unit/config/test_arguments.py @@ -0,0 +1,195 @@ +import pytest +from flexmock import flexmock + +from borgmatic.config import arguments as module + + +def test_set_values_without_keys_bails(): + config = {'option': 'value'} + module.set_values(config=config, keys=(), value=5) + + assert config == {'option': 'value'} + + +def test_set_values_with_keys_adds_them_to_config(): + config = {'option': 'value'} + + module.set_values(config=config, keys=('foo', 'bar', 'baz'), value=5) + + assert config == {'option': 'value', 'foo': {'bar': {'baz': 5}}} + + +def test_set_values_with_one_existing_key_adds_others_to_config(): + config = {'foo': {'other': 'value'}} + + module.set_values(config=config, keys=('foo', 'bar', 'baz'), value=5) + + assert config == {'foo': {'other': 'value', 'bar': {'baz': 5}}} + + +def test_set_values_with_two_existing_keys_adds_others_to_config(): + config = {'foo': {'bar': {'other': 'value'}}} + + module.set_values(config=config, keys=('foo', 'bar', 'baz'), value=5) + + assert config == {'foo': {'bar': {'other': 'value', 'baz': 5}}} + + +def test_set_values_with_list_index_key_adds_it_to_config(): + config = {'foo': {'bar': [{'option': 'value'}, {'other': 'thing'}]}} + + module.set_values(config=config, keys=('foo', 'bar[1]', 'baz'), value=5) + + assert config == {'foo': {'bar': [{'option': 'value'}, {'other': 'thing', 'baz': 5}]}} + + +def test_set_values_with_list_index_key_out_of_range_raises(): + config = {'foo': {'bar': [{'option': 'value'}]}} + + with pytest.raises(ValueError): + module.set_values(config=config, keys=('foo', 'bar[1]', 'baz'), value=5) + + +def test_set_values_with_list_index_key_missing_list_and_out_of_range_raises(): + config = {'other': 'value'} + + with pytest.raises(ValueError): + module.set_values(config=config, keys=('foo', 'bar[1]', 'baz'), value=5) + + +def test_set_values_with_final_list_index_key_adds_it_to_config(): + config = {'foo': {'bar': [1, 2]}} + + module.set_values(config=config, keys=('foo', 'bar[1]'), value=5) + + assert config == {'foo': {'bar': [1, 5]}} + + +def test_type_for_option_with_option_finds_type(): + assert ( + module.type_for_option( + schema={'type': 'object', 'properties': {'foo': {'type': 'integer'}}}, + option_keys=('foo',), + ) + == 'integer' + ) + + +def test_type_for_option_with_nested_option_finds_type(): + assert ( + module.type_for_option( + schema={ + 'type': 'object', + 'properties': { + 'foo': {'type': 'object', 'properties': {'bar': {'type': 'boolean'}}} + }, + }, + option_keys=('foo', 'bar'), + ) + == 'boolean' + ) + + +def test_type_for_option_with_missing_nested_option_finds_nothing(): + assert ( + module.type_for_option( + schema={ + 'type': 'object', + 'properties': { + 'foo': {'type': 'object', 'properties': {'other': {'type': 'integer'}}} + }, + }, + option_keys=('foo', 'bar'), + ) + is None + ) + + +def test_type_for_option_with_typeless_nested_option_finds_nothing(): + assert ( + module.type_for_option( + schema={ + 'type': 'object', + 'properties': {'foo': {'type': 'object', 'properties': {'bar': {'example': 5}}}}, + }, + option_keys=('foo', 'bar'), + ) + is None + ) + + +def test_type_for_list_index_option_finds_type(): + assert ( + module.type_for_option( + schema={ + 'type': 'object', + 'properties': {'foo': {'type': 'array', 'items': {'type': 'integer'}}}, + }, + option_keys=('foo[0]',), + ) + == 'integer' + ) + + +def test_type_for_nested_list_index_option_finds_type(): + assert ( + module.type_for_option( + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'array', + 'items': {'type': 'object', 'properties': {'bar': {'type': 'integer'}}}, + } + }, + }, + option_keys=('foo[0]', 'bar'), + ) + == 'integer' + ) + + +def test_prepare_arguments_for_config_converts_arguments_to_keys(): + assert module.prepare_arguments_for_config( + global_arguments=flexmock(**{'my_option.sub_option': 'value1', 'other_option': 'value2'}), + schema={ + 'type': 'object', + 'properties': { + 'my_option': {'type': 'object', 'properties': {'sub_option': {'type': 'string'}}}, + 'other_option': {'type': 'string'}, + }, + }, + ) == ( + (('my_option', 'sub_option'), 'value1'), + (('other_option',), 'value2'), + ) + + +def test_prepare_arguments_for_config_skips_option_with_none_value(): + assert module.prepare_arguments_for_config( + global_arguments=flexmock(**{'my_option.sub_option': None, 'other_option': 'value2'}), + schema={ + 'type': 'object', + 'properties': { + 'my_option': {'type': 'object', 'properties': {'sub_option': {'type': 'string'}}}, + 'other_option': {'type': 'string'}, + }, + }, + ) == ( + (('other_option',), 'value2'), + ) + + +def test_prepare_arguments_for_config_skips_option_missing_from_schema(): + assert module.prepare_arguments_for_config( + global_arguments=flexmock(**{'my_option.sub_option': 'value1', 'other_option': 'value2'}), + schema={ + 'type': 'object', + 'properties': { + 'my_option': {'type': 'object'}, + 'other_option': {'type': 'string'}, + }, + }, + ) == ( + (('other_option',), 'value2'), + ) From 7020f0530ad0d889fdf25e6484acb4e8b168bb17 Mon Sep 17 00:00:00 2001 From: Vandal <86826719+VandalByte@users.noreply.github.com> Date: Fri, 28 Mar 2025 22:22:19 +0530 Subject: [PATCH 183/226] update existing tests --- borgmatic/borg/recreate.py | 13 ++- borgmatic/commands/arguments.py | 1 + tests/unit/borg/test_recreate.py | 195 ++++++++++++++++--------------- 3 files changed, 111 insertions(+), 98 deletions(-) diff --git a/borgmatic/borg/recreate.py b/borgmatic/borg/recreate.py index ea2a2081..81d5a6ed 100644 --- a/borgmatic/borg/recreate.py +++ b/borgmatic/borg/recreate.py @@ -1,6 +1,5 @@ import logging import shlex -from datetime import datetime import borgmatic.borg.environment import borgmatic.config.paths @@ -76,10 +75,14 @@ def recreate_archive( + (('--timestamp', recreate_arguments.timestamp) if recreate_arguments.timestamp else ()) + (('--compression', compression) if compression else ()) + (('--chunker-params', chunker_params) if chunker_params else ()) - + flags.make_match_archives_flags( - recreate_arguments.match_archives or archive or config.get('match_archives'), - config.get('archive_name_format'), - local_borg_version, + + ( + flags.make_match_archives_flags( + recreate_arguments.match_archives or archive or config.get('match_archives'), + config.get('archive_name_format'), + local_borg_version, + ) + if recreate_arguments.match_archives + else () ) + (('--recompress', recreate_arguments.recompress) if recreate_arguments.recompress else ()) + exclude_flags diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 689959ce..8e468e8a 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1591,6 +1591,7 @@ def make_parsers(): '-a', '--match-archives', '--glob-archives', + dest='match_archives', metavar='PATTERN', help='Only consider archive names, hashes, or series matching this pattern', ) diff --git a/tests/unit/borg/test_recreate.py b/tests/unit/borg/test_recreate.py index 4bab00d9..aaa3c3fd 100644 --- a/tests/unit/borg/test_recreate.py +++ b/tests/unit/borg/test_recreate.py @@ -28,7 +28,17 @@ def test_recreate_archive_dry_run_skips_execution(): ).and_return(('repo::archive',)) flexmock(module.borgmatic.execute).should_receive('execute_command').never() - recreate_arguments = flexmock(repository=flexmock(), list=None, path=None) + recreate_arguments = flexmock( + repository=flexmock(), + list=None, + target=None, + comment=None, + timestamp=None, + compression=None, + chunker_params=None, + match_archives=None, + recompress=None, + ) result = module.recreate_archive( repository='repo', @@ -44,11 +54,6 @@ def test_recreate_archive_dry_run_skips_execution(): def test_recreate_calls_borg_with_required_flags(): - flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( - ('repo::archive',) - ) - flexmock(module).should_receive('write_patterns_file').and_return(None) - flexmock(module).should_receive('make_list_filter_flags').and_return(None) insert_execute_command_mock(('borg', 'recreate', 'repo::archive')) module.recreate_archive( @@ -56,7 +61,16 @@ def test_recreate_calls_borg_with_required_flags(): archive='archive', config={}, local_borg_version='1.2.3', - recreate_arguments=flexmock(path=None, list=None), + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + compression=None, + chunker_params=None, + match_archives=None, + recompress=None, + ), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -64,30 +78,7 @@ def test_recreate_calls_borg_with_required_flags(): ) -def test_recreate_calls_borg_without_archive(): - logger_mock = flexmock(module.logger) - logger_mock.should_receive('error').with_args('Please provide a valid archive name.').once() - - flexmock(module.borgmatic.execute).should_receive('execute_command').never() - - module.recreate_archive( - repository='repo', - archive=None, - config={}, - local_borg_version='1.2.3', - recreate_arguments=flexmock(path=None, list=None), - global_arguments=flexmock(dry_run=False, log_json=False), - local_path='borg', - patterns=None, - ) - - def test_recreate_with_remote_path(): - flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( - ('repo::archive',) - ) - flexmock(module).should_receive('write_patterns_file').and_return(None) - flexmock(module).should_receive('make_list_filter_flags').and_return(None) insert_execute_command_mock(('borg', 'recreate', '--remote-path', 'borg1', 'repo::archive')) module.recreate_archive( @@ -95,7 +86,16 @@ def test_recreate_with_remote_path(): archive='archive', config={}, local_borg_version='1.2.3', - recreate_arguments=flexmock(path=None, list=None), + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + compression=None, + chunker_params=None, + match_archives=None, + recompress=None, + ), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path='borg1', @@ -104,11 +104,6 @@ def test_recreate_with_remote_path(): def test_recreate_with_lock_wait(): - flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( - ('repo::archive',) - ) - flexmock(module).should_receive('write_patterns_file').and_return(None) - flexmock(module).should_receive('make_list_filter_flags').and_return(None) insert_execute_command_mock(('borg', 'recreate', '--lock-wait', '5', 'repo::archive')) module.recreate_archive( @@ -116,7 +111,16 @@ def test_recreate_with_lock_wait(): archive='archive', config={'lock_wait': '5'}, local_borg_version='1.2.3', - recreate_arguments=flexmock(path=None, list=None), + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + compression=None, + chunker_params=None, + match_archives=None, + recompress=None, + ), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', patterns=None, @@ -124,11 +128,6 @@ def test_recreate_with_lock_wait(): def test_recreate_with_log_info(): - flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( - ('repo::archive',) - ) - flexmock(module).should_receive('write_patterns_file').and_return(None) - flexmock(module).should_receive('make_list_filter_flags').and_return(None) insert_execute_command_mock(('borg', 'recreate', '--info', 'repo::archive')) insert_logging_mock(logging.INFO) @@ -138,7 +137,16 @@ def test_recreate_with_log_info(): archive='archive', config={}, local_borg_version='1.2.3', - recreate_arguments=flexmock(path=None, list=None), + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + compression=None, + chunker_params=None, + match_archives=None, + recompress=None, + ), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', patterns=None, @@ -146,11 +154,6 @@ def test_recreate_with_log_info(): def test_recreate_with_log_debug(): - flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( - ('repo::archive',) - ) - flexmock(module).should_receive('write_patterns_file').and_return(None) - flexmock(module).should_receive('make_list_filter_flags').and_return(None) insert_execute_command_mock(('borg', 'recreate', '--debug', '--show-rc', 'repo::archive')) insert_logging_mock(logging.DEBUG) @@ -159,7 +162,16 @@ def test_recreate_with_log_debug(): archive='archive', config={}, local_borg_version='1.2.3', - recreate_arguments=flexmock(path=None, list=None), + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + compression=None, + chunker_params=None, + match_archives=None, + recompress=None, + ), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', patterns=None, @@ -167,11 +179,6 @@ def test_recreate_with_log_debug(): def test_recreate_with_log_json(): - flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( - ('repo::archive',) - ) - flexmock(module).should_receive('write_patterns_file').and_return(None) - flexmock(module).should_receive('make_list_filter_flags').and_return(None) insert_execute_command_mock(('borg', 'recreate', '--log-json', 'repo::archive')) module.recreate_archive( @@ -179,38 +186,23 @@ def test_recreate_with_log_json(): archive='archive', config={}, local_borg_version='1.2.3', - recreate_arguments=flexmock(path=None, list=None), + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + compression=None, + chunker_params=None, + match_archives=None, + recompress=None, + ), global_arguments=flexmock(dry_run=False, log_json=True), local_path='borg', patterns=None, ) -def test_recreate_with_path_flag(): - flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( - ('repo::archive',) - ) - flexmock(module).should_receive('write_patterns_file').and_return(None) - flexmock(module).should_receive('make_list_filter_flags').and_return(None) - insert_execute_command_mock(('borg', 'recreate', '--path', '/some/path', 'repo::archive')) - - module.recreate_archive( - repository='repo', - archive='archive', - config={}, - local_borg_version='1.2.3', - recreate_arguments=flexmock(path='/some/path', list=None), - global_arguments=flexmock(dry_run=False, log_json=False), - local_path='borg', - patterns=None, - ) - - def test_recreate_with_list_filter_flags(): - flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( - ('repo::archive',) - ) - flexmock(module).should_receive('write_patterns_file').and_return(None) flexmock(module).should_receive('make_list_filter_flags').and_return('AME+-') insert_execute_command_mock( ('borg', 'recreate', '--list', '--filter', 'AME+-', 'repo::archive') @@ -221,7 +213,16 @@ def test_recreate_with_list_filter_flags(): archive='archive', config={}, local_borg_version='1.2.3', - recreate_arguments=flexmock(path=None, list=True), + recreate_arguments=flexmock( + list=True, + target=None, + comment=None, + timestamp=None, + compression=None, + chunker_params=None, + match_archives=None, + recompress=None, + ), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', patterns=None, @@ -229,12 +230,8 @@ def test_recreate_with_list_filter_flags(): def test_recreate_with_patterns_from_flag(): - flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( - ('repo::archive',) - ) mock_patterns_file = flexmock(name='patterns_file') flexmock(module).should_receive('write_patterns_file').and_return(mock_patterns_file) - flexmock(module).should_receive('make_list_filter_flags').and_return(None) insert_execute_command_mock( ('borg', 'recreate', '--patterns-from', 'patterns_file', 'repo::archive') ) @@ -244,7 +241,16 @@ def test_recreate_with_patterns_from_flag(): archive='archive', config={}, local_borg_version='1.2.3', - recreate_arguments=flexmock(path=None, list=None), + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + compression=None, + chunker_params=None, + match_archives=None, + recompress=None, + ), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', patterns=['pattern1', 'pattern2'], @@ -252,14 +258,7 @@ def test_recreate_with_patterns_from_flag(): def test_recreate_with_exclude_flags(): - flexmock(module.borgmatic.borg.flags).should_receive( - 'make_repository_archive_flags' - ).and_return(('repo::archive',)) - flexmock(module).should_receive('write_patterns_file').and_return(None) - flexmock(module).should_receive('make_list_filter_flags').and_return(None) - # Mock the make_exclude_flags to return a sample exclude flag flexmock(module).should_receive('make_exclude_flags').and_return(('--exclude', 'pattern')) - insert_execute_command_mock(('borg', 'recreate', '--exclude', 'pattern', 'repo::archive')) module.recreate_archive( @@ -267,8 +266,18 @@ def test_recreate_with_exclude_flags(): archive='archive', config={'exclude_patterns': ['pattern']}, local_borg_version='1.2.3', - recreate_arguments=flexmock(path=None, list=None), + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + compression=None, + chunker_params=None, + match_archives=None, + recompress=None, + ), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', patterns=None, ) + From 975a6e4540203d078a2ffbadf3a8524e90c573a1 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 28 Mar 2025 11:37:48 -0700 Subject: [PATCH 184/226] Add additional tests for complete coverage (#303). --- borgmatic/borg/create.py | 2 +- borgmatic/commands/arguments.py | 5 +- borgmatic/config/schema.py | 22 +++--- borgmatic/config/validate.py | 2 +- tests/integration/commands/test_arguments.py | 7 +- tests/integration/config/test_arguments.py | 1 - tests/unit/config/test_arguments.py | 8 +- tests/unit/config/test_generate.py | 1 - tests/unit/config/test_normalize.py | 5 ++ tests/unit/config/test_schema.py | 83 ++++++++++++++++++++ 10 files changed, 113 insertions(+), 23 deletions(-) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 54a7f9ad..ccdb41a9 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -196,7 +196,7 @@ def check_all_root_patterns_exist(patterns): if missing_paths: raise ValueError( - f"Source directories / root pattern paths do not exist: {', '.join(missing_paths)}" + f"Source directories or root pattern paths do not exist: {', '.join(missing_paths)}" ) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 140dc1ae..7ad600d9 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1,7 +1,6 @@ import collections import io import itertools -import json import re import sys from argparse import ArgumentParser @@ -377,7 +376,7 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): registry_name for registry_name, action_type in arguments_group._registries['action'].items() # Not using isinstance() here because we only want an exact match—no parent classes. - if type(argument_action) == action_type + if type(argument_action) is action_type ) except StopIteration: return @@ -492,7 +491,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names # As a UX nicety, allow boolean options that have a default of false to have command-line flags # without values. - if schema_type == 'boolean' and schema.get('default') == False: + if schema_type == 'boolean' and schema.get('default') is False: arguments_group.add_argument( full_flag_name, action='store_true', diff --git a/borgmatic/config/schema.py b/borgmatic/config/schema.py index 6981dde5..3d13c0c2 100644 --- a/borgmatic/config/schema.py +++ b/borgmatic/config/schema.py @@ -23,6 +23,16 @@ def get_properties(schema): return schema.get('properties', {}) +SCHEMA_TYPE_TO_PYTHON_TYPE = { + 'array': list, + 'boolean': bool, + 'integer': int, + 'number': decimal.Decimal, + 'object': dict, + 'string': str, +} + + def parse_type(schema_type, **overrides): ''' Given a schema type as a string, return the corresponding Python type. @@ -34,14 +44,7 @@ def parse_type(schema_type, **overrides): ''' try: return dict( - { - 'array': list, - 'boolean': bool, - 'integer': int, - 'number': decimal.Decimal, - 'object': dict, - 'string': str, - }, + SCHEMA_TYPE_TO_PYTHON_TYPE, **overrides, )[schema_type] except KeyError: @@ -54,7 +57,8 @@ def compare_types(schema_type, target_types, match=any): target type strings, return whether every schema type is in the set of target types. If the schema type is a list of strings, use the given match function (such as any or all) to - compare elements. + compare elements. For instance, if match is given as all, then every element of the schema_type + list must be in the target types. ''' if isinstance(schema_type, list): if match(element_schema_type in target_types for element_schema_type in schema_type): diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 29215f91..98bf080c 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -21,7 +21,7 @@ def schema_filename(): return schema_path -def load_schema(schema_path): +def load_schema(schema_path): # pragma: no cover ''' Given a schema filename path, load the schema and return it as a dict. diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 032b9495..ed6a9892 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -14,7 +14,12 @@ def test_make_argument_description_with_array_adds_example(): }, flag_name='flag', ) - == 'Thing. Example value: "[1, \'- foo\', bar: baz]"' + # Apparently different versions of ruamel.yaml serialize this + # differently. + in ( + 'Thing. Example value: "[1, \'- foo\', bar: baz]"' + 'Thing. Example value: "[1, \'- foo\', {bar: baz}]"' + ) ) diff --git a/tests/integration/config/test_arguments.py b/tests/integration/config/test_arguments.py index 449964ee..9bcc37ff 100644 --- a/tests/integration/config/test_arguments.py +++ b/tests/integration/config/test_arguments.py @@ -1,5 +1,4 @@ import pytest -from flexmock import flexmock from borgmatic.config import arguments as module diff --git a/tests/unit/config/test_arguments.py b/tests/unit/config/test_arguments.py index 25c8873f..18ad7f56 100644 --- a/tests/unit/config/test_arguments.py +++ b/tests/unit/config/test_arguments.py @@ -175,9 +175,7 @@ def test_prepare_arguments_for_config_skips_option_with_none_value(): 'other_option': {'type': 'string'}, }, }, - ) == ( - (('other_option',), 'value2'), - ) + ) == ((('other_option',), 'value2'),) def test_prepare_arguments_for_config_skips_option_missing_from_schema(): @@ -190,6 +188,4 @@ def test_prepare_arguments_for_config_skips_option_missing_from_schema(): 'other_option': {'type': 'string'}, }, }, - ) == ( - (('other_option',), 'value2'), - ) + ) == ((('other_option',), 'value2'),) diff --git a/tests/unit/config/test_generate.py b/tests/unit/config/test_generate.py index 885bdcca..ad90bb5a 100644 --- a/tests/unit/config/test_generate.py +++ b/tests/unit/config/test_generate.py @@ -2,7 +2,6 @@ import pytest from flexmock import flexmock from borgmatic.config import generate as module -import borgmatic.config.schema def test_schema_to_sample_configuration_generates_config_map_with_examples(): diff --git a/tests/unit/config/test_normalize.py b/tests/unit/config/test_normalize.py index abd7e54d..29fe25e8 100644 --- a/tests/unit/config/test_normalize.py +++ b/tests/unit/config/test_normalize.py @@ -359,6 +359,11 @@ def test_normalize_commands_moves_individual_command_hooks_to_unified_commands( {'repositories': [{'path': '/repo', 'label': 'foo'}]}, False, ), + ( + {'repositories': [{'path': None, 'label': 'foo'}]}, + {'repositories': []}, + False, + ), ( {'prefix': 'foo'}, {'prefix': 'foo'}, diff --git a/tests/unit/config/test_schema.py b/tests/unit/config/test_schema.py index a07b19ba..8af890fb 100644 --- a/tests/unit/config/test_schema.py +++ b/tests/unit/config/test_schema.py @@ -1,3 +1,5 @@ +import pytest + from borgmatic.config import schema as module @@ -75,3 +77,84 @@ def test_get_properties_interleaves_oneof_list_properties(): ('field3', {'example': 'Example 3'}), ] ) + + +def test_parse_type_maps_schema_type_to_python_type(): + module.parse_type('boolean') == bool + + +def test_parse_type_with_unknown_schema_type_raises(): + with pytest.raises(ValueError): + module.parse_type('what') + + +def test_parse_type_respect_overrides_when_mapping_types(): + module.parse_type('boolean', boolean=int) == int + + +@pytest.mark.parametrize( + 'schema_type,target_types,match,expected_result', + ( + ( + 'string', + {'integer', 'string', 'boolean'}, + None, + True, + ), + ( + 'string', + {'integer', 'boolean'}, + None, + False, + ), + ( + 'string', + {'integer', 'string', 'boolean'}, + all, + True, + ), + ( + 'string', + {'integer', 'boolean'}, + all, + False, + ), + ( + ['string', 'array'], + {'integer', 'string', 'boolean'}, + None, + True, + ), + ( + ['string', 'array'], + {'integer', 'boolean'}, + None, + False, + ), + ( + ['string', 'array'], + {'integer', 'string', 'boolean', 'array'}, + all, + True, + ), + ( + ['string', 'array'], + {'integer', 'string', 'boolean'}, + all, + False, + ), + ( + ['string', 'array'], + {'integer', 'boolean'}, + all, + False, + ), + ), +) +def test_compare_types_returns_whether_schema_type_matches_target_types( + schema_type, target_types, match, expected_result +): + if match: + assert module.compare_types(schema_type, target_types, match) == expected_result + else: + assert module.compare_types(schema_type, target_types) == expected_result From 8b3a682edfa833383f53ef8b2f1dd9b6b5e08baf Mon Sep 17 00:00:00 2001 From: Vandal <86826719+VandalByte@users.noreply.github.com> Date: Sat, 29 Mar 2025 01:26:20 +0530 Subject: [PATCH 185/226] add tests and minor fixes --- borgmatic/borg/recreate.py | 15 +- borgmatic/commands/arguments.py | 12 -- tests/unit/borg/test_recreate.py | 269 +++++++++++++++++++++++++++---- 3 files changed, 243 insertions(+), 53 deletions(-) diff --git a/borgmatic/borg/recreate.py b/borgmatic/borg/recreate.py index 81d5a6ed..bea11b58 100644 --- a/borgmatic/borg/recreate.py +++ b/borgmatic/borg/recreate.py @@ -33,6 +33,8 @@ def recreate_archive( exclude_flags = make_exclude_flags(config) compression = config.get('compression', None) chunker_params = config.get('chunker_params', None) + # Available recompress MODES: 'if-different' (default), 'always', 'never' + recompress = config.get('recompress', None) # Write patterns to a temporary file and use that file with --patterns-from. patterns_file = write_patterns_file( @@ -42,11 +44,6 @@ def recreate_archive( recreate_command = ( (local_path, 'recreate') + (('--remote-path', remote_path) if remote_path else ()) - # + ( - # ('--path', recreate_arguments.path) - # if recreate_arguments.path - # else () - # ) + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait is not None else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) @@ -62,11 +59,7 @@ def recreate_archive( else () ) # Flag --target works only for a single archive - + ( - ('--target', recreate_arguments.target) - if recreate_arguments.target and recreate_arguments.archive - else () - ) + + (('--target', recreate_arguments.target) if recreate_arguments.target and archive else ()) + ( ('--comment', shlex.quote(recreate_arguments.comment)) if recreate_arguments.comment @@ -84,7 +77,7 @@ def recreate_archive( if recreate_arguments.match_archives else () ) - + (('--recompress', recreate_arguments.recompress) if recreate_arguments.recompress else ()) + + (('--recompress', recompress) if recompress else ()) + exclude_flags + ( flags.make_repository_archive_flags(repository, archive, local_borg_version) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 8e468e8a..b0185348 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1562,13 +1562,6 @@ def make_parsers(): '--archive', help='Archive name, hash, or series to recreate', ) - # recreate_group.add_argument( - # '--path', - # metavar='PATH', - # dest='path', - # help='Path to recreate the repository/archive', - # required=True, - # ) recreate_group.add_argument( '--list', dest='list', action='store_true', help='Show per-file details' ) @@ -1595,11 +1588,6 @@ def make_parsers(): metavar='PATTERN', help='Only consider archive names, hashes, or series matching this pattern', ) - recreate_group.add_argument( - '--recompress', - metavar='MODE', - help='Recompress data chunks according to MODE: [if-different (default), always, never]', - ) recreate_group.add_argument( '-h', '--help', action='help', help='Show this help message and exit' ) diff --git a/tests/unit/borg/test_recreate.py b/tests/unit/borg/test_recreate.py index aaa3c3fd..5ec0eeaf 100644 --- a/tests/unit/borg/test_recreate.py +++ b/tests/unit/borg/test_recreate.py @@ -1,4 +1,5 @@ import logging +import shlex from flexmock import flexmock @@ -34,10 +35,7 @@ def test_recreate_archive_dry_run_skips_execution(): target=None, comment=None, timestamp=None, - compression=None, - chunker_params=None, match_archives=None, - recompress=None, ) result = module.recreate_archive( @@ -66,10 +64,7 @@ def test_recreate_calls_borg_with_required_flags(): target=None, comment=None, timestamp=None, - compression=None, - chunker_params=None, match_archives=None, - recompress=None, ), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', @@ -91,10 +86,7 @@ def test_recreate_with_remote_path(): target=None, comment=None, timestamp=None, - compression=None, - chunker_params=None, match_archives=None, - recompress=None, ), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', @@ -116,10 +108,7 @@ def test_recreate_with_lock_wait(): target=None, comment=None, timestamp=None, - compression=None, - chunker_params=None, match_archives=None, - recompress=None, ), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', @@ -142,10 +131,7 @@ def test_recreate_with_log_info(): target=None, comment=None, timestamp=None, - compression=None, - chunker_params=None, match_archives=None, - recompress=None, ), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', @@ -167,10 +153,7 @@ def test_recreate_with_log_debug(): target=None, comment=None, timestamp=None, - compression=None, - chunker_params=None, match_archives=None, - recompress=None, ), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', @@ -191,10 +174,7 @@ def test_recreate_with_log_json(): target=None, comment=None, timestamp=None, - compression=None, - chunker_params=None, match_archives=None, - recompress=None, ), global_arguments=flexmock(dry_run=False, log_json=True), local_path='borg', @@ -218,10 +198,7 @@ def test_recreate_with_list_filter_flags(): target=None, comment=None, timestamp=None, - compression=None, - chunker_params=None, match_archives=None, - recompress=None, ), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', @@ -246,10 +223,7 @@ def test_recreate_with_patterns_from_flag(): target=None, comment=None, timestamp=None, - compression=None, - chunker_params=None, match_archives=None, - recompress=None, ), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', @@ -271,13 +245,248 @@ def test_recreate_with_exclude_flags(): target=None, comment=None, timestamp=None, - compression=None, - chunker_params=None, match_archives=None, - recompress=None, ), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', patterns=None, ) + +def test_recreate_with_target_flag(): + insert_execute_command_mock(('borg', 'recreate', '--target', 'new-archive', 'repo::archive')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target='new-archive', + comment=None, + timestamp=None, + match_archives=None, + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_comment_flag(): + insert_execute_command_mock( + ('borg', 'recreate', '--comment', shlex.quote('This is a test comment'), 'repo::archive') + ) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment='This is a test comment', + timestamp=None, + match_archives=None, + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_timestamp_flag(): + insert_execute_command_mock( + ('borg', 'recreate', '--timestamp', '2023-10-01T12:00:00', 'repo::archive') + ) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp='2023-10-01T12:00:00', + match_archives=None, + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_compression_flag(): + insert_execute_command_mock(('borg', 'recreate', '--compression', 'lz4', 'repo::archive')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={'compression': 'lz4'}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives=None, + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_chunker_params_flag(): + insert_execute_command_mock( + ('borg', 'recreate', '--chunker-params', '19,23,21,4095', 'repo::archive') + ) + + module.recreate_archive( + repository='repo', + archive='archive', + config={'chunker_params': '19,23,21,4095'}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives=None, + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_recompress_flag(): + insert_execute_command_mock(('borg', 'recreate', '--recompress', 'always', 'repo::archive')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={'recompress': 'always'}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives=None, + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_match_archives_star(): + insert_execute_command_mock(('borg', 'recreate', 'repo::archive')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives='*', + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_match_archives_regex(): + insert_execute_command_mock(('borg', 'recreate', 'repo::archive')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives='re:.*', + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_match_archives_shell(): + insert_execute_command_mock(('borg', 'recreate', 'repo::archive')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives='sh:*', + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_glob_archives_flag(): + insert_execute_command_mock(('borg', 'recreate', '--glob-archives', 'foo-*', 'repo::archive')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives='foo-*', + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_match_archives_flag(): + insert_execute_command_mock( + ('borg', 'recreate', '--match-archives', 'sh:foo-*', '--repo', 'repo', 'archive') + ) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='2.0.0b3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives='sh:foo-*', + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) From 1c27e0dadc40ce34138a6397243d69bb0a93660e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 28 Mar 2025 13:46:58 -0700 Subject: [PATCH 186/226] Add an end-to-end test for command-line flags of configuration options (#303). --- tests/end-to-end/test_config_flag.py | 51 ++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 tests/end-to-end/test_config_flag.py diff --git a/tests/end-to-end/test_config_flag.py b/tests/end-to-end/test_config_flag.py new file mode 100644 index 00000000..0300ac4e --- /dev/null +++ b/tests/end-to-end/test_config_flag.py @@ -0,0 +1,51 @@ +import os +import shlex +import shutil +import subprocess +import tempfile + + +def generate_configuration(config_path): + ''' + Generate borgmatic configuration into a file at the config path, and update the defaults so as + to work for testing (including injecting the given repository path and tacking on an encryption + passphrase). But don't actually set the repository path, as that's done on the command-line + below. + ''' + subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' ')) + config = ( + open(config_path) + .read() + .replace('- ssh://user@backupserver/./{fqdn}', '') # noqa: FS003 + .replace('- /var/local/backups/local.borg', '') + .replace('- /home/user/path with spaces', '') + .replace('- /home', f'- {config_path}') + .replace('- /etc', '') + .replace('- /var/log/syslog*', '') + + 'encryption_passphrase: "test"' + ) + config_file = open(config_path, 'w') + config_file.write(config) + config_file.close() + + +def test_config_flags_do_not_error(): + temporary_directory = tempfile.mkdtemp() + repository_path = os.path.join(temporary_directory, 'test.borg') + + original_working_directory = os.getcwd() + + try: + config_path = os.path.join(temporary_directory, 'test.yaml') + generate_configuration(config_path) + + subprocess.check_call( + shlex.split(f'borgmatic -v 2 --config {config_path} --repositories "[{{path: {repository_path}, label: repo}}]" repo-create --encryption repokey') + ) + + subprocess.check_call( + shlex.split(f'borgmatic create --config {config_path} --repositories[0].path "{repository_path}"') + ) + finally: + os.chdir(original_working_directory) + shutil.rmtree(temporary_directory) From 0182dbd914edce4b577230929677803621af5ee6 Mon Sep 17 00:00:00 2001 From: Gautam Aggarwal Date: Sat, 29 Mar 2025 03:43:58 +0000 Subject: [PATCH 187/226] Added 2 new unit tests and updated docs --- docs/how-to/set-up-backups.md | 10 +++++++++ tests/unit/commands/test_borgmatic.py | 30 +++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index c90b9136..d63368f3 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -296,6 +296,16 @@ skip_actions: - compact ``` +## Disabling default actions + +By default, running `borgmatic` without any arguments will perform the default backup actions (create, prune, and compact). If you want to disable this behavior and require explicit actions to be specified, add the following to your configuration: + +```yaml +default_actions: false +``` + +With this setting, running `borgmatic` without arguments will show the help message instead of performing any actions. + ## Autopilot diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 15512a67..27e91cf5 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -2099,3 +2099,33 @@ def test_check_and_show_help_on_no_args_does_not_show_help_when_args_provided(): flexmock(module.sys).should_receive('exit').never() module.check_and_show_help_on_no_args({'test.yaml': {'default_actions': False}}) +def test_check_and_show_help_on_no_args_with_no_default_actions_in_all_configs(): + flexmock(module.sys).should_receive('argv').and_return(['borgmatic']) + + # Both configs have default_actions set to False, so help should be shown + configs = { + 'config1.yaml': {'default_actions': False}, + 'config2.yaml': {'default_actions': False} + } + + # Expect help to be shown + flexmock(module).should_receive('parse_arguments').with_args('--help').once() + flexmock(module.sys).should_receive('exit').with_args(0).once() + + module.check_and_show_help_on_no_args(configs) + +def test_check_and_show_help_on_no_args_with_conflicting_configs(): + flexmock(module.sys).should_receive('argv').and_return(['borgmatic']) + + # Simulate two config files with conflicting 'default_actions' values + configs = { + 'config1.yaml': {'default_actions': True}, + 'config2.yaml': {'default_actions': False} + } + + # Expect help not to be shown because at least one config enables default actions + flexmock(module).should_receive('parse_arguments').never() + flexmock(module.sys).should_receive('exit').never() + + module.check_and_show_help_on_no_args(configs) + From 668f767bfc8b48e1c73e2471f2282e9e61643d36 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 28 Mar 2025 23:11:15 -0700 Subject: [PATCH 188/226] Adding some missing tests and fixing related flag vs. config logic (#303). --- borgmatic/actions/compact.py | 12 ++- borgmatic/actions/create.py | 18 +++- borgmatic/actions/export_tar.py | 6 +- borgmatic/actions/extract.py | 6 +- borgmatic/actions/repo_create.py | 18 +++- borgmatic/borg/check.py | 4 +- borgmatic/borg/delete.py | 7 +- borgmatic/borg/prune.py | 11 ++- borgmatic/borg/transfer.py | 9 +- borgmatic/commands/arguments.py | 8 +- borgmatic/config/schema.yaml | 4 +- tests/end-to-end/test_config_flag.py | 8 +- tests/unit/actions/test_compact.py | 78 +++++++++++++++++ tests/unit/actions/test_create.py | 112 +++++++++++++++++++++++++ tests/unit/actions/test_export_tar.py | 80 ++++++++++++++++++ tests/unit/actions/test_extract.py | 78 +++++++++++++++++ tests/unit/actions/test_repo_create.py | 88 +++++++++++++++++++ tests/unit/borg/test_check.py | 39 ++++++++- tests/unit/borg/test_delete.py | 36 +++++++- tests/unit/borg/test_prune.py | 50 ++++++++++- tests/unit/borg/test_transfer.py | 35 +++++++- 21 files changed, 671 insertions(+), 36 deletions(-) diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index 7386d230..dbc2e573 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -37,9 +37,17 @@ def run_compact( global_arguments, local_path=local_path, remote_path=remote_path, - progress=compact_arguments.progress or config.get('progress'), + progress=( + config.get('progress') + if compact_arguments.progress is None + else compact_arguments.progress + ), cleanup_commits=compact_arguments.cleanup_commits, - threshold=compact_arguments.threshold or config.get('compact_threshold'), + threshold=( + config.get('compact_threshold') + if compact_arguments.threshold is None + else compact_arguments.threshold + ), ) else: # pragma: nocover logger.info('Skipping compact (only available/needed in Borg 1.2+)') diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index c771f3d2..e0ef3b1f 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -327,10 +327,22 @@ def run_create( borgmatic_runtime_directory, local_path=local_path, remote_path=remote_path, - progress=create_arguments.progress or config.get('progress'), - stats=create_arguments.stats or config.get('stats'), + progress=( + config.get('progress') + if create_arguments.progress is None + else create_arguments.progress + ), + stats=( + config.get('statistics') + if create_arguments.stats is None + else create_arguments.stats + ), json=create_arguments.json, - list_files=create_arguments.list_files or config.get('list'), + list_files=( + config.get('list_details') + if create_arguments.list_files is None + else create_arguments.list_files + ), stream_processes=stream_processes, ) diff --git a/borgmatic/actions/export_tar.py b/borgmatic/actions/export_tar.py index 1b41548b..cc18aeeb 100644 --- a/borgmatic/actions/export_tar.py +++ b/borgmatic/actions/export_tar.py @@ -43,6 +43,10 @@ def run_export_tar( local_path=local_path, remote_path=remote_path, tar_filter=export_tar_arguments.tar_filter, - list_files=export_tar_arguments.list_files or config.get('list'), + list_files=( + config.get('list_details') + if export_tar_arguments.list_files is None + else export_tar_arguments.list_files + ), strip_components=export_tar_arguments.strip_components, ) diff --git a/borgmatic/actions/extract.py b/borgmatic/actions/extract.py index 83f81176..3e42b57d 100644 --- a/borgmatic/actions/extract.py +++ b/borgmatic/actions/extract.py @@ -45,5 +45,9 @@ def run_extract( remote_path=remote_path, destination_path=extract_arguments.destination, strip_components=extract_arguments.strip_components, - progress=extract_arguments.progress or config.get('progress'), + progress=( + config.get('progress') + if extract_arguments.progress is None + else extract_arguments.progress + ), ) diff --git a/borgmatic/actions/repo_create.py b/borgmatic/actions/repo_create.py index 49f4e6c0..5166b9b8 100644 --- a/borgmatic/actions/repo_create.py +++ b/borgmatic/actions/repo_create.py @@ -41,9 +41,21 @@ def run_repo_create( encryption_mode, repo_create_arguments.source_repository, repo_create_arguments.copy_crypt_key, - repo_create_arguments.append_only or repository.get('append_only'), - repo_create_arguments.storage_quota or repository.get('storage_quota'), - repo_create_arguments.make_parent_dirs or repository.get('make_parent_dirs'), + ( + repository.get('append_only') + if repo_create_arguments.append_only is None + else repo_create_arguments.append_only + ), + ( + repository.get('storage_quota') + if repo_create_arguments.storage_quota is None + else repo_create_arguments.storage_quota + ), + ( + repository.get('make_parent_dirs') + if repo_create_arguments.make_parent_dirs is None + else repo_create_arguments.make_parent_dirs + ), local_path=local_path, remote_path=remote_path, ) diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 13f4ffca..c02b6ec4 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -143,7 +143,9 @@ def check_archives( umask = config.get('umask') borg_exit_codes = config.get('borg_exit_codes') working_directory = borgmatic.config.paths.get_working_directory(config) - progress = check_arguments.progress or config.get('progress') + progress = ( + config.get('progress') if check_arguments.progress is None else check_arguments.progress + ) if 'data' in checks: checks.add('archives') diff --git a/borgmatic/borg/delete.py b/borgmatic/borg/delete.py index a9c3478c..f31730ce 100644 --- a/borgmatic/borg/delete.py +++ b/borgmatic/borg/delete.py @@ -35,7 +35,12 @@ def make_delete_command( + borgmatic.borg.flags.make_flags('log-json', global_arguments.log_json) + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait')) + borgmatic.borg.flags.make_flags( - 'list', delete_arguments.list_archives or config.get('list') + 'list', + ( + config.get('list_details') + if delete_arguments.list_archives is None + else delete_arguments.list_archives + ), ) + ( (('--force',) + (('--force',) if delete_arguments.force >= 2 else ())) diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index 5eef3a29..b29cbe67 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -66,7 +66,12 @@ def prune_archives( borgmatic.logger.add_custom_log_levels() umask = config.get('umask', None) lock_wait = config.get('lock_wait', None) - stats = prune_arguments.stats or config.get('stats') + stats = config.get('statistics') if prune_arguments.stats is None else prune_arguments.stats + list_archives = ( + config.get('list_details') + if prune_arguments.list_archives is None + else prune_arguments.list_archives + ) extra_borg_options = config.get('extra_borg_options', {}).get('prune', '') full_command = ( @@ -88,14 +93,14 @@ def prune_archives( prune_arguments, excludes=('repository', 'match_archives', 'stats', 'list_archives'), ) - + (('--list',) if prune_arguments.list_archives or config.get('list') else ()) + + (('--list',) if list_archives else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (('--dry-run',) if dry_run else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + flags.make_repository_flags(repository_path, local_borg_version) ) - if stats or prune_arguments.list_archives: + if stats or list_archives: output_log_level = logging.ANSWER else: output_log_level = logging.INFO diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index ca90063e..528cd64c 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -52,13 +52,16 @@ def transfer_archives( + flags.make_flags('other-repo', transfer_arguments.source_repository) + flags.make_flags('dry-run', dry_run) ) + progress = ( + config.get('progress') + if transfer_arguments.progress is None + else transfer_arguments.progress + ) return execute_command( full_command, output_log_level=logging.ANSWER, - output_file=( - DO_NOT_CAPTURE if (transfer_arguments.progress or config.get('progress')) else None - ), + output_file=DO_NOT_CAPTURE if progress else None, environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 7ad600d9..6344a0b2 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -287,7 +287,7 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse ) -OMITTED_FLAG_NAMES = {'match_archives', 'progress', 'stats', 'list'} +OMITTED_FLAG_NAMES = {'match_archives', 'progress', 'statistics', 'list_files'} def make_argument_description(schema, flag_name): @@ -520,9 +520,9 @@ def make_parsers(schema, unparsed_arguments): config_paths = collect.get_default_config_paths(expand_home=True) unexpanded_config_paths = collect.get_default_config_paths(expand_home=False) - # allow_abbrev=False prevents the global parser from erroring about "ambiguous" options like - # --encryption. Such options are intended for an action parser rather than the global parser, - # and so we don't want to error on them here. + # Using allow_abbrev=False here prevents the global parser from erroring about "ambiguous" + # options like --encryption. Such options are intended for an action parser rather than the + # global parser, and so we don't want to error on them here. global_parser = ArgumentParser(allow_abbrev=False, add_help=False) global_group = global_parser.add_argument_group('global arguments') diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index aaeb4476..d0bcaa0f 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -823,7 +823,7 @@ properties: actions. Defaults to false. default: false example: true - stats: + statistics: type: boolean description: | Display statistics for an archive when running supported actions. @@ -831,7 +831,7 @@ properties: false. default: false example: true - list: + list_details: type: boolean description: | Display details for each file or archive as it is processed when diff --git a/tests/end-to-end/test_config_flag.py b/tests/end-to-end/test_config_flag.py index 0300ac4e..279a1c6e 100644 --- a/tests/end-to-end/test_config_flag.py +++ b/tests/end-to-end/test_config_flag.py @@ -40,11 +40,15 @@ def test_config_flags_do_not_error(): generate_configuration(config_path) subprocess.check_call( - shlex.split(f'borgmatic -v 2 --config {config_path} --repositories "[{{path: {repository_path}, label: repo}}]" repo-create --encryption repokey') + shlex.split( + f'borgmatic -v 2 --config {config_path} --repositories "[{{path: {repository_path}, label: repo}}]" repo-create --encryption repokey' + ) ) subprocess.check_call( - shlex.split(f'borgmatic create --config {config_path} --repositories[0].path "{repository_path}"') + shlex.split( + f'borgmatic create --config {config_path} --repositories[0].path "{repository_path}"' + ) ) finally: os.chdir(original_working_directory) diff --git a/tests/unit/actions/test_compact.py b/tests/unit/actions/test_compact.py index 14dcfaad..1531fd59 100644 --- a/tests/unit/actions/test_compact.py +++ b/tests/unit/actions/test_compact.py @@ -51,6 +51,84 @@ def test_compact_runs_with_selected_repository(): ) +def test_compact_favors_flags_over_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive( + 'repositories_match' + ).once().and_return(True) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.compact).should_receive('compact_segments').with_args( + object, + object, + object, + object, + object, + local_path=object, + remote_path=object, + progress=False, + cleanup_commits=object, + threshold=15, + ).once() + compact_arguments = flexmock( + repository=flexmock(), + progress=False, + cleanup_commits=flexmock(), + threshold=15, + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_compact( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={'progress': True, 'compact_threshold': 20}, + local_borg_version=None, + compact_arguments=compact_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + + +def test_compact_favors_defaults_to_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive( + 'repositories_match' + ).once().and_return(True) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.compact).should_receive('compact_segments').with_args( + object, + object, + object, + object, + object, + local_path=object, + remote_path=object, + progress=True, + cleanup_commits=object, + threshold=20, + ).once() + compact_arguments = flexmock( + repository=flexmock(), + progress=None, + cleanup_commits=flexmock(), + threshold=None, + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_compact( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={'progress': True, 'compact_threshold': 20}, + local_borg_version=None, + compact_arguments=compact_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + + def test_compact_bails_if_repository_does_not_match(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 1f6ca7dc..6f40fb50 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -506,6 +506,118 @@ def test_run_create_runs_with_selected_repository(): ) +def test_run_create_favors_flags_over_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive( + 'repositories_match' + ).once().and_return(True) + flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return( + flexmock() + ) + flexmock(module.borgmatic.borg.create).should_receive('create_archive').with_args( + object, + object, + object, + object, + object, + object, + object, + local_path=object, + remote_path=object, + progress=False, + stats=False, + json=object, + list_files=False, + stream_processes=object, + ).once() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({}) + flexmock(module.borgmatic.hooks.dispatch).should_receive( + 'call_hooks_even_if_unconfigured' + ).and_return({}) + flexmock(module).should_receive('collect_patterns').and_return(()) + flexmock(module).should_receive('process_patterns').and_return([]) + flexmock(module.os.path).should_receive('join').and_return('/run/borgmatic/bootstrap') + create_arguments = flexmock( + repository=flexmock(), + progress=False, + stats=False, + json=False, + list_files=False, + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + list( + module.run_create( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={'progress': True, 'statistics': True, 'list_details': True}, + config_paths=['/tmp/test.yaml'], + local_borg_version=None, + create_arguments=create_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + ) + + +def test_run_create_defaults_to_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive( + 'repositories_match' + ).once().and_return(True) + flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return( + flexmock() + ) + flexmock(module.borgmatic.borg.create).should_receive('create_archive').with_args( + object, + object, + object, + object, + object, + object, + object, + local_path=object, + remote_path=object, + progress=True, + stats=True, + json=object, + list_files=True, + stream_processes=object, + ).once() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({}) + flexmock(module.borgmatic.hooks.dispatch).should_receive( + 'call_hooks_even_if_unconfigured' + ).and_return({}) + flexmock(module).should_receive('collect_patterns').and_return(()) + flexmock(module).should_receive('process_patterns').and_return([]) + flexmock(module.os.path).should_receive('join').and_return('/run/borgmatic/bootstrap') + create_arguments = flexmock( + repository=flexmock(), + progress=True, + stats=True, + json=False, + list_files=True, + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + list( + module.run_create( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={'progress': True, 'statistics': True, 'list_details': True}, + config_paths=['/tmp/test.yaml'], + local_borg_version=None, + create_arguments=create_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + ) + + def test_run_create_bails_if_repository_does_not_match(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive( diff --git a/tests/unit/actions/test_export_tar.py b/tests/unit/actions/test_export_tar.py index aea54af3..b2e91c38 100644 --- a/tests/unit/actions/test_export_tar.py +++ b/tests/unit/actions/test_export_tar.py @@ -27,3 +27,83 @@ def test_run_export_tar_does_not_raise(): local_path=None, remote_path=None, ) + + +def test_run_export_tar_favors_flags_over_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.export_tar).should_receive('export_tar_archive').with_args( + object, + object, + object, + object, + object, + object, + object, + object, + local_path=object, + remote_path=object, + tar_filter=object, + list_files=False, + strip_components=object, + ).once() + export_tar_arguments = flexmock( + repository=flexmock(), + archive=flexmock(), + paths=flexmock(), + destination=flexmock(), + tar_filter=flexmock(), + list_files=False, + strip_components=flexmock(), + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_export_tar( + repository={'path': 'repo'}, + config={'list_details': True}, + local_borg_version=None, + export_tar_arguments=export_tar_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) + + +def test_run_export_tar_defaults_to_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.export_tar).should_receive('export_tar_archive').with_args( + object, + object, + object, + object, + object, + object, + object, + object, + local_path=object, + remote_path=object, + tar_filter=object, + list_files=True, + strip_components=object, + ).once() + export_tar_arguments = flexmock( + repository=flexmock(), + archive=flexmock(), + paths=flexmock(), + destination=flexmock(), + tar_filter=flexmock(), + list_files=None, + strip_components=flexmock(), + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_export_tar( + repository={'path': 'repo'}, + config={'list_details': True}, + local_borg_version=None, + export_tar_arguments=export_tar_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/actions/test_extract.py b/tests/unit/actions/test_extract.py index c483adcc..3707c887 100644 --- a/tests/unit/actions/test_extract.py +++ b/tests/unit/actions/test_extract.py @@ -27,3 +27,81 @@ def test_run_extract_calls_hooks(): local_path=None, remote_path=None, ) + + +def test_run_extract_favors_flags_over_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').with_args( + object, + object, + object, + object, + object, + object, + object, + local_path=object, + remote_path=object, + destination_path=object, + strip_components=object, + progress=False, + ).once() + extract_arguments = flexmock( + paths=flexmock(), + progress=False, + destination=flexmock(), + strip_components=flexmock(), + archive=flexmock(), + repository='repo', + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_extract( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={'repositories': ['repo'], 'progress': True}, + local_borg_version=None, + extract_arguments=extract_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) + + +def test_run_extract_defaults_to_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').with_args( + object, + object, + object, + object, + object, + object, + object, + local_path=object, + remote_path=object, + destination_path=object, + strip_components=object, + progress=True, + ).once() + extract_arguments = flexmock( + paths=flexmock(), + progress=None, + destination=flexmock(), + strip_components=flexmock(), + archive=flexmock(), + repository='repo', + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_extract( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={'repositories': ['repo'], 'progress': True}, + local_borg_version=None, + extract_arguments=extract_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/actions/test_repo_create.py b/tests/unit/actions/test_repo_create.py index 035dd623..665d9db7 100644 --- a/tests/unit/actions/test_repo_create.py +++ b/tests/unit/actions/test_repo_create.py @@ -105,3 +105,91 @@ def test_run_repo_create_bails_if_repository_does_not_match(): local_path=None, remote_path=None, ) + + +def test_run_repo_create_favors_flags_over_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.repo_create).should_receive('create_repository').with_args( + object, + object, + object, + object, + object, + object, + object, + object, + append_only=False, + storage_quota=0, + make_parent_dirs=False, + local_path=object, + remote_path=object, + ).once() + arguments = flexmock( + encryption_mode=flexmock(), + source_repository=flexmock(), + repository=flexmock(), + copy_crypt_key=flexmock(), + append_only=False, + storage_quota=0, + make_parent_dirs=False, + ) + + module.run_repo_create( + repository={ + 'path': 'repo', + 'append_only': True, + 'storage_quota': '10G', + 'make_parent_dirs': True, + }, + config={}, + local_borg_version=None, + repo_create_arguments=arguments, + global_arguments=flexmock(dry_run=False), + local_path=None, + remote_path=None, + ) + + +def test_run_repo_create_defaults_to_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.repo_create).should_receive('create_repository').with_args( + object, + object, + object, + object, + object, + object, + object, + object, + append_only=True, + storage_quota='10G', + make_parent_dirs=True, + local_path=object, + remote_path=object, + ).once() + arguments = flexmock( + encryption_mode=flexmock(), + source_repository=flexmock(), + repository=flexmock(), + copy_crypt_key=flexmock(), + append_only=None, + storage_quota=None, + make_parent_dirs=None, + ) + + module.run_repo_create( + repository={ + 'path': 'repo', + 'append_only': True, + 'storage_quota': '10G', + 'make_parent_dirs': True, + }, + config={}, + local_borg_version=None, + repo_create_arguments=arguments, + global_arguments=flexmock(dry_run=False), + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index cb3ce67c..80ce0d94 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -331,8 +331,8 @@ def test_get_repository_id_with_missing_json_keys_raises(): ) -def test_check_archives_with_progress_passes_through_to_borg(): - config = {} +def test_check_archives_favors_progress_flag_over_config(): + config = {'progress': False} flexmock(module).should_receive('make_check_name_flags').with_args( {'repository'}, () ).and_return(()) @@ -366,6 +366,41 @@ def test_check_archives_with_progress_passes_through_to_borg(): ) +def test_check_archives_defaults_to_progress_config(): + config = {'progress': True} + flexmock(module).should_receive('make_check_name_flags').with_args( + {'repository'}, () + ).and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.environment).should_receive('make_environment') + flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'check', '--progress', 'repo'), + output_file=module.DO_NOT_CAPTURE, + environment=None, + working_directory=None, + borg_local_path='borg', + borg_exit_codes=None, + ).once() + + module.check_archives( + repository_path='repo', + config=config, + local_borg_version='1.2.3', + check_arguments=flexmock( + progress=None, + repair=None, + only_checks=None, + force=None, + match_archives=None, + max_duration=None, + ), + global_arguments=flexmock(log_json=False), + checks={'repository'}, + archive_filter_flags=(), + ) + + def test_check_archives_with_repair_passes_through_to_borg(): config = {} flexmock(module).should_receive('make_check_name_flags').with_args( diff --git a/tests/unit/borg/test_delete.py b/tests/unit/borg/test_delete.py index 6d8614b6..f6ace21d 100644 --- a/tests/unit/borg/test_delete.py +++ b/tests/unit/borg/test_delete.py @@ -171,7 +171,7 @@ def test_make_delete_command_includes_lock_wait(): assert command == ('borg', 'delete', '--lock-wait', '5', 'repo') -def test_make_delete_command_includes_list(): +def test_make_delete_command_favors_list_flag_over_config(): flexmock(module.borgmatic.borg.flags).should_receive('make_flags').and_return(()) flexmock(module.borgmatic.borg.flags).should_receive('make_flags').with_args( 'list', True @@ -184,7 +184,7 @@ def test_make_delete_command_includes_list(): command = module.make_delete_command( repository={'path': 'repo'}, - config={}, + config={'list_details': False}, local_borg_version='1.2.3', delete_arguments=flexmock(list_archives=True, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), @@ -195,6 +195,30 @@ def test_make_delete_command_includes_list(): assert command == ('borg', 'delete', '--list', 'repo') +def test_make_delete_command_defaults_to_list_config(): + flexmock(module.borgmatic.borg.flags).should_receive('make_flags').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive('make_flags').with_args( + 'list', True + ).and_return(('--list',)) + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive('make_flags_from_arguments').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( + ('repo',) + ) + + command = module.make_delete_command( + repository={'path': 'repo'}, + config={'list_details': True}, + local_borg_version='1.2.3', + delete_arguments=flexmock(list_archives=None, force=0, match_archives=None, archive=None), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + remote_path=None, + ) + + assert command == ('borg', 'delete', '--list', 'repo') + + def test_make_delete_command_includes_force(): flexmock(module.borgmatic.borg.flags).should_receive('make_flags').and_return(()) flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) @@ -287,8 +311,12 @@ def test_make_delete_command_includes_match_archives(): assert command == ('borg', 'delete', '--match-archives', 'sh:foo*', 'repo') +LOGGING_ANSWER = flexmock() + + def test_delete_archives_with_archive_calls_borg_delete(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module.borgmatic.borg.repo_delete).should_receive('delete_repository').never() flexmock(module).should_receive('make_delete_command').and_return(flexmock()) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -308,6 +336,7 @@ def test_delete_archives_with_archive_calls_borg_delete(): def test_delete_archives_with_match_archives_calls_borg_delete(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module.borgmatic.borg.repo_delete).should_receive('delete_repository').never() flexmock(module).should_receive('make_delete_command').and_return(flexmock()) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -328,6 +357,7 @@ def test_delete_archives_with_match_archives_calls_borg_delete(): @pytest.mark.parametrize('argument_name', module.ARCHIVE_RELATED_ARGUMENT_NAMES[2:]) def test_delete_archives_with_archive_related_argument_calls_borg_delete(argument_name): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module.borgmatic.borg.repo_delete).should_receive('delete_repository').never() flexmock(module).should_receive('make_delete_command').and_return(flexmock()) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -347,6 +377,7 @@ def test_delete_archives_with_archive_related_argument_calls_borg_delete(argumen def test_delete_archives_without_archive_related_argument_calls_borg_repo_delete(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.repo_delete).should_receive('delete_repository').once() flexmock(module).should_receive('make_delete_command').never() @@ -367,6 +398,7 @@ def test_delete_archives_without_archive_related_argument_calls_borg_repo_delete def test_delete_archives_calls_borg_delete_with_working_directory(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module.borgmatic.borg.repo_delete).should_receive('delete_repository').never() command = flexmock() flexmock(module).should_receive('make_delete_command').and_return(command) diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index 3f2da33a..1d7888a3 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -361,7 +361,7 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_flags(): ) -def test_prune_archives_with_stats_calls_borg_with_stats_flag_and_answer_output_log_level(): +def test_prune_archives_favors_stats_flag_over_config(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -375,14 +375,35 @@ def test_prune_archives_with_stats_calls_borg_with_stats_flag_and_answer_output_ module.prune_archives( dry_run=False, repository_path='repo', - config={}, + config={'statistics': False}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, ) -def test_prune_archives_with_files_calls_borg_with_list_flag_and_answer_output_log_level(): +def test_prune_archives_defaults_to_stats_config(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) + insert_execute_command_mock(PRUNE_COMMAND + ('--stats', 'repo'), module.borgmatic.logger.ANSWER) + + prune_arguments = flexmock(stats=None, list_archives=False) + module.prune_archives( + dry_run=False, + repository_path='repo', + config={'statistics': True}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + prune_arguments=prune_arguments, + ) + + +def test_prune_archives_favors_list_flag_over_config(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -396,7 +417,28 @@ def test_prune_archives_with_files_calls_borg_with_list_flag_and_answer_output_l module.prune_archives( dry_run=False, repository_path='repo', - config={}, + config={'list_details': False}, + local_borg_version='1.2.3', + global_arguments=flexmock(log_json=False), + prune_arguments=prune_arguments, + ) + + +def test_prune_archives_defaults_to_list_config(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) + insert_execute_command_mock(PRUNE_COMMAND + ('--list', 'repo'), module.borgmatic.logger.ANSWER) + + prune_arguments = flexmock(stats=False, list_archives=None) + module.prune_archives( + dry_run=False, + repository_path='repo', + config={'list_details': True}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, diff --git a/tests/unit/borg/test_transfer.py b/tests/unit/borg/test_transfer.py index edd2131b..9e6618e5 100644 --- a/tests/unit/borg/test_transfer.py +++ b/tests/unit/borg/test_transfer.py @@ -436,7 +436,7 @@ def test_transfer_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): ) -def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): +def test_transfer_archives_favors_progress_flag_over_config(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) @@ -458,7 +458,7 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): module.transfer_archives( dry_run=False, repository_path='repo', - config={}, + config={'progress': False}, local_borg_version='2.3.4', transfer_arguments=flexmock( archive=None, progress=True, match_archives=None, source_repository=None @@ -467,6 +467,37 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): ) +def test_transfer_archives_defaults_to_progress_flag(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--progress',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.environment).should_receive('make_environment') + flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) + flexmock(module).should_receive('execute_command').with_args( + ('borg', 'transfer', '--progress', '--repo', 'repo'), + output_log_level=module.borgmatic.logger.ANSWER, + output_file=module.DO_NOT_CAPTURE, + environment=None, + working_directory=None, + borg_local_path='borg', + borg_exit_codes=None, + ) + + module.transfer_archives( + dry_run=False, + repository_path='repo', + config={'progress': True}, + local_borg_version='2.3.4', + transfer_arguments=flexmock( + archive=None, progress=None, match_archives=None, source_repository=None + ), + global_arguments=flexmock(log_json=False), + ) + + @pytest.mark.parametrize('argument_name', ('upgrader', 'sort_by', 'first', 'last')) def test_transfer_archives_passes_through_arguments_to_borg(argument_name): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') From 2716d9d0b0980af80a2840abff40573d703f1306 Mon Sep 17 00:00:00 2001 From: Vandal <86826719+VandalByte@users.noreply.github.com> Date: Sat, 29 Mar 2025 23:25:50 +0530 Subject: [PATCH 189/226] add to schema --- borgmatic/config/schema.yaml | 18 ++++ tests/unit/borg/test_recreate.py | 158 ++++++++++++++++++++++++++++++- 2 files changed, 173 insertions(+), 3 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 9fb8f361..9d622b31 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -284,6 +284,24 @@ properties: http://borgbackup.readthedocs.io/en/stable/usage/create.html for details. Defaults to "lz4". example: lz4 + recompress: + type: string + enum: ['if-different', 'always', 'never'] + description: | + Mode for recompressing data chunks according to MODE. + Possible modes are: + - "if-different": Recompress if the current compression + is with a different compression algorithm. + - "always": Recompress even if the current compression + is with the same compression algorithm. Use this to change + the compression level. + - "never": Do not recompress. Use this option to explicitly + prevent recompression. + See https://borgbackup.readthedocs.io/en/stable/usage/recreate.html + for details. + Not passing --recompress is equivalent to "--recompress never". + Defaults to "if-different". + example: if-different upload_rate_limit: type: integer description: | diff --git a/tests/unit/borg/test_recreate.py b/tests/unit/borg/test_recreate.py index 5ec0eeaf..fe2739b7 100644 --- a/tests/unit/borg/test_recreate.py +++ b/tests/unit/borg/test_recreate.py @@ -7,9 +7,6 @@ from borgmatic.borg import recreate as module from ..test_verbosity import insert_logging_mock -# from borgmatic.borg.pattern import Pattern, Pattern_type, Pattern_style, Pattern_source -# from borgmatic.borg.create import make_exclude_flags, make_list_filter_flags - def insert_execute_command_mock(command, working_directory=None, borg_exit_codes=None): flexmock(module.borgmatic.borg.environment).should_receive('make_environment') @@ -23,7 +20,21 @@ def insert_execute_command_mock(command, working_directory=None, borg_exit_codes ).once() +def mock_dependencies(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) + + def test_recreate_archive_dry_run_skips_execution(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) flexmock(module.borgmatic.borg.flags).should_receive( 'make_repository_archive_flags' ).and_return(('repo::archive',)) @@ -52,6 +63,13 @@ def test_recreate_archive_dry_run_skips_execution(): def test_recreate_calls_borg_with_required_flags(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) insert_execute_command_mock(('borg', 'recreate', 'repo::archive')) module.recreate_archive( @@ -74,6 +92,13 @@ def test_recreate_calls_borg_with_required_flags(): def test_recreate_with_remote_path(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) insert_execute_command_mock(('borg', 'recreate', '--remote-path', 'borg1', 'repo::archive')) module.recreate_archive( @@ -96,6 +121,13 @@ def test_recreate_with_remote_path(): def test_recreate_with_lock_wait(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) insert_execute_command_mock(('borg', 'recreate', '--lock-wait', '5', 'repo::archive')) module.recreate_archive( @@ -117,6 +149,13 @@ def test_recreate_with_lock_wait(): def test_recreate_with_log_info(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) insert_execute_command_mock(('borg', 'recreate', '--info', 'repo::archive')) insert_logging_mock(logging.INFO) @@ -140,6 +179,13 @@ def test_recreate_with_log_info(): def test_recreate_with_log_debug(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) insert_execute_command_mock(('borg', 'recreate', '--debug', '--show-rc', 'repo::archive')) insert_logging_mock(logging.DEBUG) @@ -162,6 +208,13 @@ def test_recreate_with_log_debug(): def test_recreate_with_log_json(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) insert_execute_command_mock(('borg', 'recreate', '--log-json', 'repo::archive')) module.recreate_archive( @@ -183,6 +236,12 @@ def test_recreate_with_log_json(): def test_recreate_with_list_filter_flags(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) flexmock(module).should_receive('make_list_filter_flags').and_return('AME+-') insert_execute_command_mock( ('borg', 'recreate', '--list', '--filter', 'AME+-', 'repo::archive') @@ -207,6 +266,12 @@ def test_recreate_with_list_filter_flags(): def test_recreate_with_patterns_from_flag(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) mock_patterns_file = flexmock(name='patterns_file') flexmock(module).should_receive('write_patterns_file').and_return(mock_patterns_file) insert_execute_command_mock( @@ -232,6 +297,12 @@ def test_recreate_with_patterns_from_flag(): def test_recreate_with_exclude_flags(): + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) flexmock(module).should_receive('make_exclude_flags').and_return(('--exclude', 'pattern')) insert_execute_command_mock(('borg', 'recreate', '--exclude', 'pattern', 'repo::archive')) @@ -254,6 +325,13 @@ def test_recreate_with_exclude_flags(): def test_recreate_with_target_flag(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) insert_execute_command_mock(('borg', 'recreate', '--target', 'new-archive', 'repo::archive')) module.recreate_archive( @@ -275,6 +353,13 @@ def test_recreate_with_target_flag(): def test_recreate_with_comment_flag(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) insert_execute_command_mock( ('borg', 'recreate', '--comment', shlex.quote('This is a test comment'), 'repo::archive') ) @@ -298,6 +383,13 @@ def test_recreate_with_comment_flag(): def test_recreate_with_timestamp_flag(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) insert_execute_command_mock( ('borg', 'recreate', '--timestamp', '2023-10-01T12:00:00', 'repo::archive') ) @@ -321,6 +413,13 @@ def test_recreate_with_timestamp_flag(): def test_recreate_with_compression_flag(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) insert_execute_command_mock(('borg', 'recreate', '--compression', 'lz4', 'repo::archive')) module.recreate_archive( @@ -342,6 +441,13 @@ def test_recreate_with_compression_flag(): def test_recreate_with_chunker_params_flag(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) insert_execute_command_mock( ('borg', 'recreate', '--chunker-params', '19,23,21,4095', 'repo::archive') ) @@ -365,6 +471,13 @@ def test_recreate_with_chunker_params_flag(): def test_recreate_with_recompress_flag(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) insert_execute_command_mock(('borg', 'recreate', '--recompress', 'always', 'repo::archive')) module.recreate_archive( @@ -386,6 +499,13 @@ def test_recreate_with_recompress_flag(): def test_recreate_with_match_archives_star(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) insert_execute_command_mock(('borg', 'recreate', 'repo::archive')) module.recreate_archive( @@ -407,6 +527,13 @@ def test_recreate_with_match_archives_star(): def test_recreate_with_match_archives_regex(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) insert_execute_command_mock(('borg', 'recreate', 'repo::archive')) module.recreate_archive( @@ -428,6 +555,13 @@ def test_recreate_with_match_archives_regex(): def test_recreate_with_match_archives_shell(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) insert_execute_command_mock(('borg', 'recreate', 'repo::archive')) module.recreate_archive( @@ -449,6 +583,15 @@ def test_recreate_with_match_archives_shell(): def test_recreate_with_glob_archives_flag(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return( + ('--glob-archives', 'foo-*') + ) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) insert_execute_command_mock(('borg', 'recreate', '--glob-archives', 'foo-*', 'repo::archive')) module.recreate_archive( @@ -470,6 +613,15 @@ def test_recreate_with_glob_archives_flag(): def test_recreate_with_match_archives_flag(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return( + ('--match-archives', 'sh:foo-*') + ) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('--repo', 'repo', 'archive')) insert_execute_command_mock( ('borg', 'recreate', '--match-archives', 'sh:foo-*', '--repo', 'repo', 'archive') ) From f6929f8891aac09d9d62a32b70d542d0177f5ebf Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 29 Mar 2025 14:26:54 -0700 Subject: [PATCH 190/226] Add last couple of missing tests after audit (#303). --- NEWS | 4 ++-- borgmatic/commands/completion/flag.py | 4 ++-- tests/unit/commands/completion/test_flag.py | 20 ++++++++++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 tests/unit/commands/completion/test_flag.py diff --git a/NEWS b/NEWS index 0d2507c1..7af92c69 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,6 @@ 2.0.0.dev0 - * #303: Add command-line flags for every borgmatic configuration option. See the - documentation for more information: + * #303: Deprecate the "--override" flag in favor of direct command-line flags for every borgmatic + configuration option. See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides * #303: Add configuration options that serve as defaults for some (but not all) command-line action flags. For example, each entry in "repositories:" now has an "encryption" option that diff --git a/borgmatic/commands/completion/flag.py b/borgmatic/commands/completion/flag.py index 6de6517f..5a6bc797 100644 --- a/borgmatic/commands/completion/flag.py +++ b/borgmatic/commands/completion/flag.py @@ -1,7 +1,7 @@ def variants(flag_name): ''' - Given an flag name as a string, yield it and any variations that should be complete-able as - well. For instance, for a string like "--foo[0].bar", yield "--foo[0].bar", "--foo[1].bar", ..., + Given a flag name as a string, yield it and any variations that should be complete-able as well. + For instance, for a string like "--foo[0].bar", yield "--foo[0].bar", "--foo[1].bar", ..., "--foo[9].bar". ''' if '[0]' in flag_name: diff --git a/tests/unit/commands/completion/test_flag.py b/tests/unit/commands/completion/test_flag.py new file mode 100644 index 00000000..06e99979 --- /dev/null +++ b/tests/unit/commands/completion/test_flag.py @@ -0,0 +1,20 @@ +from borgmatic.commands.completion import flag as module + + +def test_variants_passes_through_non_list_index_flag_name(): + assert tuple(module.variants('foo')) == ('foo',) + + +def test_variants_broadcasts_list_index_flag_name_with_a_range_of_indices(): + assert tuple(module.variants('foo[0].bar')) == ( + 'foo[0].bar', + 'foo[1].bar', + 'foo[2].bar', + 'foo[3].bar', + 'foo[4].bar', + 'foo[5].bar', + 'foo[6].bar', + 'foo[7].bar', + 'foo[8].bar', + 'foo[9].bar', + ) From dbf2e78f6253757793f6639df711fec9aa6a95b2 Mon Sep 17 00:00:00 2001 From: Vandal <86826719+VandalByte@users.noreply.github.com> Date: Sun, 30 Mar 2025 03:05:46 +0530 Subject: [PATCH 191/226] help changes --- borgmatic/commands/arguments.py | 6 +++--- borgmatic/config/schema.yaml | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index b0185348..6269a4e0 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1568,17 +1568,17 @@ def make_parsers(): recreate_group.add_argument( '--target', metavar='TARGET', - help='Name of new archive name', + help='Create a new archive from the specified archive (via --archive), without replacing it', ) recreate_group.add_argument( '--comment', metavar='COMMENT', - help='Add a comment text to the archive, if archive not provided, consider all archives', + help='Add a comment text to the archive or, if an archive is not provided, to all matching archives', ) recreate_group.add_argument( '--timestamp', metavar='TIMESTAMP', - help='Manually specify the archive creation date/time (UTC)', + help='Manually override the archive creation date/time (UTC)', ) recreate_group.add_argument( '-a', diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 9d622b31..f3b98f36 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -299,8 +299,7 @@ properties: prevent recompression. See https://borgbackup.readthedocs.io/en/stable/usage/recreate.html for details. - Not passing --recompress is equivalent to "--recompress never". - Defaults to "if-different". + Defaults to "never". example: if-different upload_rate_limit: type: integer From da324ebeb7034596c589b87808eb1335e7e634a8 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 29 Mar 2025 15:15:36 -0700 Subject: [PATCH 192/226] Add "recreate" action to NEWS and docs (#610). --- NEWS | 2 ++ borgmatic/commands/arguments.py | 48 ++++++++++++++++----------------- borgmatic/config/schema.yaml | 9 +++---- docs/Dockerfile | 2 +- 4 files changed, 31 insertions(+), 30 deletions(-) diff --git a/NEWS b/NEWS index df2261d8..52c38ca1 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,8 @@ 2.0.0.dev0 * #345: Add a "key import" action to import a repository key from backup. * #422: Add home directory expansion to file-based and KeePassXC credential hooks. + * #610: Add a "recreate" action for recreating archives, for instance for retroactively excluding + particular files from existing archives. * #790, #821: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more flexible "commands:". See the documentation for more information: https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 6269a4e0..5b900d47 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1522,30 +1522,6 @@ def make_parsers(): '-h', '--help', action='help', help='Show this help message and exit' ) - borg_parser = action_parsers.add_parser( - 'borg', - aliases=ACTION_ALIASES['borg'], - help='Run an arbitrary Borg command', - description="Run an arbitrary Borg command based on borgmatic's configuration", - add_help=False, - ) - borg_group = borg_parser.add_argument_group('borg arguments') - borg_group.add_argument( - '--repository', - help='Path of repository to pass to Borg, defaults to the configured repositories, quoted globs supported', - ) - borg_group.add_argument( - '--archive', help='Archive name, hash, or series to pass to Borg (or "latest")' - ) - borg_group.add_argument( - '--', - metavar='OPTION', - dest='options', - nargs='+', - help='Options to pass to Borg, command first ("create", "list", etc). "--" is optional. To specify the repository or the archive, you must use --repository or --archive instead of providing them here.', - ) - borg_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') - recreate_parser = action_parsers.add_parser( 'recreate', aliases=ACTION_ALIASES['recreate'], @@ -1592,6 +1568,30 @@ def make_parsers(): '-h', '--help', action='help', help='Show this help message and exit' ) + borg_parser = action_parsers.add_parser( + 'borg', + aliases=ACTION_ALIASES['borg'], + help='Run an arbitrary Borg command', + description="Run an arbitrary Borg command based on borgmatic's configuration", + add_help=False, + ) + borg_group = borg_parser.add_argument_group('borg arguments') + borg_group.add_argument( + '--repository', + help='Path of repository to pass to Borg, defaults to the configured repositories, quoted globs supported', + ) + borg_group.add_argument( + '--archive', help='Archive name, hash, or series to pass to Borg (or "latest")' + ) + borg_group.add_argument( + '--', + metavar='OPTION', + dest='options', + nargs='+', + help='Options to pass to Borg, command first ("create", "list", etc). "--" is optional. To specify the repository or the archive, you must use --repository or --archive instead of providing them here.', + ) + borg_group.add_argument('-h', '--help', action='help', help='Show this help message and exit') + return global_parser, action_parsers, global_plus_action_parser diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 3154ddad..7dd4e51c 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -290,16 +290,15 @@ properties: description: | Mode for recompressing data chunks according to MODE. Possible modes are: - - "if-different": Recompress if the current compression + * "if-different": Recompress if the current compression is with a different compression algorithm. - - "always": Recompress even if the current compression + * "always": Recompress even if the current compression is with the same compression algorithm. Use this to change the compression level. - - "never": Do not recompress. Use this option to explicitly + * "never": Do not recompress. Use this option to explicitly prevent recompression. See https://borgbackup.readthedocs.io/en/stable/usage/recreate.html - for details. - Defaults to "never". + for details. Defaults to "never". example: if-different upload_rate_limit: type: integer diff --git a/docs/Dockerfile b/docs/Dockerfile index afa3457d..cfa1b828 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -4,7 +4,7 @@ COPY . /app RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib RUN pip install --break-system-packages --no-cache /app && borgmatic config generate && chmod +r /etc/borgmatic/config.yaml RUN borgmatic --help > /command-line.txt \ - && for action in repo-create transfer create prune compact check delete extract config "config bootstrap" "config generate" "config validate" export-tar mount umount repo-delete restore repo-list list repo-info info break-lock "key export" "key import" "key change-passphrase" borg; do \ + && for action in repo-create transfer create prune compact check delete extract config "config bootstrap" "config generate" "config validate" export-tar mount umount repo-delete restore repo-list list repo-info info break-lock "key export" "key import" "key change-passphrase" recreate borg; do \ echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \ && borgmatic $action --help >> /command-line.txt; done RUN /app/docs/fetch-contributors >> /contributors.html From 8101e5c56fc1d49024a962effb8fd7a7312bf07a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 29 Mar 2025 15:24:37 -0700 Subject: [PATCH 193/226] Add "list_details" config option support to new "recreate" action (#303). --- borgmatic/borg/recreate.py | 19 +++++++------- borgmatic/commands/arguments.py | 2 +- tests/unit/borg/test_recreate.py | 44 +++++++++++++++++++++++--------- 3 files changed, 43 insertions(+), 22 deletions(-) diff --git a/borgmatic/borg/recreate.py b/borgmatic/borg/recreate.py index bea11b58..15969f5f 100644 --- a/borgmatic/borg/recreate.py +++ b/borgmatic/borg/recreate.py @@ -22,24 +22,25 @@ def recreate_archive( patterns=None, ): ''' - Given a local or remote repository path, an archive name, a configuration dict, - the local Borg version string, an argparse.Namespace of recreate arguments, - an argparse.Namespace of global arguments, optional local and remote Borg paths. - - Executes the recreate command with the given arguments. + Given a local or remote repository path, an archive name, a configuration dict, the local Borg + version string, an argparse.Namespace of recreate arguments, an argparse.Namespace of global + arguments, optional local and remote Borg paths, executes the recreate command with the given + arguments. ''' - lock_wait = config.get('lock_wait', None) exclude_flags = make_exclude_flags(config) compression = config.get('compression', None) chunker_params = config.get('chunker_params', None) - # Available recompress MODES: 'if-different' (default), 'always', 'never' + # Available recompress MODES: "if-different", "always", "never" (default) recompress = config.get('recompress', None) # Write patterns to a temporary file and use that file with --patterns-from. patterns_file = write_patterns_file( patterns, borgmatic.config.paths.get_working_directory(config) ) + list_files = ( + config.get('list_details') if recreate_arguments.list is None else recreate_arguments.list + ) recreate_command = ( (local_path, 'recreate') @@ -55,10 +56,10 @@ def recreate_archive( '--filter', make_list_filter_flags(local_borg_version, global_arguments.dry_run), ) - if recreate_arguments.list + if list_files else () ) - # Flag --target works only for a single archive + # Flag --target works only for a single archive. + (('--target', recreate_arguments.target) if recreate_arguments.target and archive else ()) + ( ('--comment', shlex.quote(recreate_arguments.comment)) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 8e3241d7..b4623a10 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -288,7 +288,7 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse ) -OMITTED_FLAG_NAMES = {'match_archives', 'progress', 'statistics', 'list_files'} +OMITTED_FLAG_NAMES = {'match_archives', 'progress', 'statistics', 'list_details'} def make_argument_description(schema, flag_name): diff --git a/tests/unit/borg/test_recreate.py b/tests/unit/borg/test_recreate.py index fe2739b7..dd91218d 100644 --- a/tests/unit/borg/test_recreate.py +++ b/tests/unit/borg/test_recreate.py @@ -20,16 +20,6 @@ def insert_execute_command_mock(command, working_directory=None, borg_exit_codes ).once() -def mock_dependencies(): - flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) - flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) - flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') - flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) - flexmock(module.borgmatic.borg.flags).should_receive( - 'make_repository_archive_flags' - ).and_return(('repo::archive',)) - - def test_recreate_archive_dry_run_skips_execution(): flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) @@ -235,7 +225,7 @@ def test_recreate_with_log_json(): ) -def test_recreate_with_list_filter_flags(): +def test_recreate_archive_favors_list_flag_over_config(): flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) @@ -250,7 +240,7 @@ def test_recreate_with_list_filter_flags(): module.recreate_archive( repository='repo', archive='archive', - config={}, + config={'list_details': False}, local_borg_version='1.2.3', recreate_arguments=flexmock( list=True, @@ -265,6 +255,36 @@ def test_recreate_with_list_filter_flags(): ) +def test_recreate_archive_defaults_to_list_config(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) + flexmock(module).should_receive('make_list_filter_flags').and_return('AME+-') + insert_execute_command_mock( + ('borg', 'recreate', '--list', '--filter', 'AME+-', 'repo::archive') + ) + + module.recreate_archive( + repository='repo', + archive='archive', + config={'list_details': True}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives=None, + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + def test_recreate_with_patterns_from_flag(): flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') From 36265eea7dab22dd6e9fb081d49d4f7b3f78bf3b Mon Sep 17 00:00:00 2001 From: Gautam Aggarwal Date: Sun, 30 Mar 2025 01:34:30 +0000 Subject: [PATCH 194/226] Docs update --- borgmatic/config/schema.yaml | 6 +----- docs/how-to/set-up-backups.md | 10 +++++++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 7a4c4a28..c3f8e1a9 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -2670,10 +2670,6 @@ properties: description: | Whether to apply default actions (e.g., backup) when no arguments are supplied to the borgmatic command. If set to true, borgmatic - will trigger the default actions,which include : - -Create backups based on your configured schedules and paths. - - Prune old backups based on your retention policies. - - Verify the integrity of your backups. - + will trigger the default actions(create, prune, compact and check). If set to false, borgmatic will display the help message instead. example: true diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index d63368f3..5871bd6b 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -296,15 +296,19 @@ skip_actions: - compact ``` -## Disabling default actions +### Disabling default actions -By default, running `borgmatic` without any arguments will perform the default backup actions (create, prune, and compact). If you want to disable this behavior and require explicit actions to be specified, add the following to your configuration: +By default, running `borgmatic` without any arguments will perform the default +backup actions (create, prune, compact and check). If you want to disable this +behavior and require explicit actions to be specified, add the following to +your configuration: ```yaml default_actions: false ``` -With this setting, running `borgmatic` without arguments will show the help message instead of performing any actions. +With this setting, running `borgmatic` without arguments will show the help +message instead of performing any actions. ## Autopilot From 65d1b9235d4d27155270f90defb696d1ff7ecc47 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 29 Mar 2025 19:02:11 -0700 Subject: [PATCH 195/226] Add "default_actions" to NEWS (#262). --- NEWS | 2 ++ borgmatic/config/schema.yaml | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 52c38ca1..9e5a3e4e 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,6 @@ 2.0.0.dev0 + * #262: Add a "default_actions" option that supports disabling default actions when borgmatic is + run without any command-line arguments. * #345: Add a "key import" action to import a repository key from backup. * #422: Add home directory expansion to file-based and KeePassXC credential hooks. * #610: Add a "recreate" action for recreating archives, for instance for retroactively excluding diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index aaabe1e5..757820ac 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -2690,6 +2690,6 @@ properties: description: | Whether to apply default actions (e.g., backup) when no arguments are supplied to the borgmatic command. If set to true, borgmatic - will trigger the default actions(create, prune, compact and check). - If set to false, borgmatic will display the help message instead. + triggers the default actions (create, prune, compact and check). If + set to false, borgmatic displays the help message instead. example: true From 5716e61f8f872ff7a2e1aac5507290e7e0f994f5 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 29 Mar 2025 19:54:40 -0700 Subject: [PATCH 196/226] Code formatting (#262). --- borgmatic/commands/borgmatic.py | 4 +--- tests/unit/commands/test_borgmatic.py | 21 ++++++++++++--------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 8bcdf82a..f3281008 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -961,9 +961,7 @@ def check_and_show_help_on_no_args(configs): default backup behavior. """ if len(sys.argv) == 1: # No arguments provided - default_actions = any( - config.get('default_actions', True) for config in configs.values() - ) + default_actions = any(config.get('default_actions', True) for config in configs.values()) if not default_actions: parse_arguments('--help') sys.exit(0) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 296bcbbf..3abdff87 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -2120,6 +2120,8 @@ def test_collect_configuration_run_summary_logs_outputs_merged_json_results(): arguments=arguments, ) ) + + def test_check_and_show_help_on_no_args_shows_help_when_no_args_and_default_actions_false(): flexmock(module.sys).should_receive('argv').and_return(['borgmatic']) flexmock(module).should_receive('parse_arguments').with_args('--help').once() @@ -2140,33 +2142,34 @@ def test_check_and_show_help_on_no_args_does_not_show_help_when_args_provided(): flexmock(module.sys).should_receive('exit').never() module.check_and_show_help_on_no_args({'test.yaml': {'default_actions': False}}) + def test_check_and_show_help_on_no_args_with_no_default_actions_in_all_configs(): flexmock(module.sys).should_receive('argv').and_return(['borgmatic']) - + # Both configs have default_actions set to False, so help should be shown configs = { 'config1.yaml': {'default_actions': False}, - 'config2.yaml': {'default_actions': False} + 'config2.yaml': {'default_actions': False}, } - + # Expect help to be shown flexmock(module).should_receive('parse_arguments').with_args('--help').once() flexmock(module.sys).should_receive('exit').with_args(0).once() - + module.check_and_show_help_on_no_args(configs) + def test_check_and_show_help_on_no_args_with_conflicting_configs(): flexmock(module.sys).should_receive('argv').and_return(['borgmatic']) - + # Simulate two config files with conflicting 'default_actions' values configs = { 'config1.yaml': {'default_actions': True}, - 'config2.yaml': {'default_actions': False} + 'config2.yaml': {'default_actions': False}, } - + # Expect help not to be shown because at least one config enables default actions flexmock(module).should_receive('parse_arguments').never() flexmock(module.sys).should_receive('exit').never() - - module.check_and_show_help_on_no_args(configs) + module.check_and_show_help_on_no_args(configs) From fd8c11eb0a02bac3e76ebce235fd9b124e1cf9e8 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 29 Mar 2025 21:59:47 -0700 Subject: [PATCH 197/226] Add documentation for "native" command-line overrides without --override (#303). --- docs/how-to/make-per-application-backups.md | 79 ++++++++++++++++++--- 1 file changed, 71 insertions(+), 8 deletions(-) diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 095fef5f..db381999 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -482,16 +482,83 @@ applications, but then set the repository for each application at runtime. Or you might want to try a variant of an option for testing purposes without actually touching your configuration file. +New in version 2.0.0 Whatever the reason, you can override borgmatic configuration options at the -command-line via the `--override` flag. Here's an example: +command-line, as there's a command-line flag corresponding to every +configuration option (with its underscores converted to dashes). + +For instance, to override the `compression` configuration option, use the +corresponding `--compression` flag on the command-line: + +```bash +borgmatic create --compression zstd +``` + +What this does is load your given configuration files and for each one, disregard +the configured value for the `compression` option and use the value given on the +command-line instead—but just for the duration of the borgmatic run. + +You can override nested configuration options too by separating such option +names with a period. For instance: + +```bash +borgmatic create --bootstrap.store-config-files false +``` + +You can even set complex option data structures by using inline YAML syntax. For +example, set the `repositories` option with a YAML list of key/value pairs: + +```bash +borgmatic create --repositories "[{path: /mnt/backup, label: local}]" +``` + +If your override value contains characters like colons or spaces, then you'll +need to use quotes for it to parse correctly. + +You can also set individual nested options within existing list elements: + +```bash +borgmatic create --repositories[0].path /mnt/backup +``` + +This updates the `path` option for the first repository in `repositories`. +Change the `[0]` index as needed to address different list elements. And note +that this only works for elements already set in configuration; you can't append +new list elements from the command-line. + +See the [command-line reference +documentation](https://torsion.org/borgmatic/docs/reference/command-line/) for +the full set of available arguments, including examples of each for the complex +values. + +There are a handful of configuration options that don't have corresponding +command-line flags at the global scope, but instead have flags within individual +borgmatic actions. For instance, the `list_details` option can be overridden by +the `--list` flag that's only present on particular actions. Similarly with +`progress` and `--progress`, `statistics` and `--stats`, and `match_archives` +and `--match-archives`. + +An alternate to command-line overrides is passing in your values via +[environment +variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/). + + +### Deprecated overrides + +Prior to version 2.0.0 +Configuration overrides were performed with an `--override` flag. You can still +use `--override` with borgmatic 2.0.0+, but it's deprecated in favor of the new +command-line flags described above. + +Here's an example of `--override`: ```bash borgmatic create --override remote_path=/usr/local/bin/borg1 ``` -What this does is load your configuration files and for each one, disregard -the configured value for the `remote_path` option and use the value of -`/usr/local/bin/borg1` instead. +What this does is load your given configuration files and for each one, disregard +the configured value for the `remote_path` option and use the value given on the +command-line instead—but just for the duration of the borgmatic run. You can even override nested values or multiple values at once. For instance: @@ -540,10 +607,6 @@ reference](https://torsion.org/borgmatic/docs/reference/configuration/) for which options are list types. (YAML list values look like `- this` with an indentation and a leading dash.) -An alternate to command-line overrides is passing in your values via -[environment -variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/). - ## Constant interpolation From 5cea1e1b724f56bbc7a493052fc4345a9f2a43f0 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 29 Mar 2025 22:52:17 -0700 Subject: [PATCH 198/226] Fix flake error (#262). --- borgmatic/commands/borgmatic.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index f3281008..20ce8bca 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -955,11 +955,11 @@ def exit_with_help_link(): # pragma: no cover def check_and_show_help_on_no_args(configs): - """ - Check if the 'borgmatic' command is run without any arguments. If the configuration option - 'default_actions' is set to False, show the help message. Otherwise, trigger the - default backup behavior. - """ + ''' + Check if the borgmatic command is run without any arguments. If the configuration option + "default_actions" is set to False, show the help message. Otherwise, trigger the default backup + behavior. + ''' if len(sys.argv) == 1: # No arguments provided default_actions = any(config.get('default_actions', True) for config in configs.values()) if not default_actions: From f7e4d38762df534857817d00d02ed52579322d01 Mon Sep 17 00:00:00 2001 From: Gautam Aggarwal Date: Sun, 30 Mar 2025 14:02:56 +0000 Subject: [PATCH 199/226] First Draft --- borgmatic/hooks/credential/keepassxc.py | 26 ++++---- tests/unit/hooks/credential/test_keepassxc.py | 65 +++++++++++++++++++ 2 files changed, 79 insertions(+), 12 deletions(-) diff --git a/borgmatic/hooks/credential/keepassxc.py b/borgmatic/hooks/credential/keepassxc.py index 7e6ac913..1a0da572 100644 --- a/borgmatic/hooks/credential/keepassxc.py +++ b/borgmatic/hooks/credential/keepassxc.py @@ -11,27 +11,23 @@ def load_credential(hook_config, config, credential_parameters): ''' Given the hook configuration dict, the configuration dict, and a credential parameters tuple containing a KeePassXC database path and an attribute name to load, run keepassxc-cli to fetch - the corresponidng KeePassXC credential and return it. + the corresponding KeePassXC credential and return it. Raise ValueError if keepassxc-cli can't retrieve the credential. ''' try: - (database_path, attribute_name) = credential_parameters + database_path, attribute_name = credential_parameters[:2] + extra_args = credential_parameters[2:] # Handle additional arguments like --key-file or --yubikey except ValueError: - path_and_name = ' '.join(credential_parameters) - - raise ValueError( - f'Cannot load credential with invalid KeePassXC database path and attribute name: "{path_and_name}"' - ) + raise ValueError( f'Invalid KeePassXC credential parameters: {credential_parameters}') expanded_database_path = os.path.expanduser(database_path) if not os.path.exists(expanded_database_path): - raise ValueError( - f'Cannot load credential because KeePassXC database path does not exist: {database_path}' - ) + raise ValueError( f'KeePassXC database path does not exist: {database_path}') - return borgmatic.execute.execute_command_and_capture_output( + # Build the keepassxc-cli command + command = ( tuple(shlex.split((hook_config or {}).get('keepassxc_cli_command', 'keepassxc-cli'))) + ( 'show', @@ -41,4 +37,10 @@ def load_credential(hook_config, config, credential_parameters): expanded_database_path, attribute_name, ) - ).rstrip(os.linesep) + + tuple(extra_args) # Append extra arguments + ) + + try: + return borgmatic.execute.execute_command_and_capture_output(command).rstrip(os.linesep) + except Exception as e: + raise ValueError(f'Failed to retrieve credential: {e}') diff --git a/tests/unit/hooks/credential/test_keepassxc.py b/tests/unit/hooks/credential/test_keepassxc.py index 0c460e23..9fddf133 100644 --- a/tests/unit/hooks/credential/test_keepassxc.py +++ b/tests/unit/hooks/credential/test_keepassxc.py @@ -116,3 +116,68 @@ def test_load_credential_with_expanded_directory_with_present_database_fetches_p ) == 'password' ) + + +def test_load_credential_with_key_file(): + flexmock(module.os.path).should_receive('expanduser').with_args('database.kdbx').and_return( + 'database.kdbx' + ) + flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).with_args( + ( + 'keepassxc-cli', + 'show', + '--show-protected', + '--attributes', + 'Password', + 'database.kdbx', + 'mypassword', + '--key-file', + '/path/to/keyfile', + ) + ).and_return( + 'password' + ).once() + + assert ( + module.load_credential( + hook_config={}, + config={}, + credential_parameters=('database.kdbx', 'mypassword', '--key-file', '/path/to/keyfile'), + ) + == 'password' + ) + + +def test_load_credential_with_yubikey(): + flexmock(module.os.path).should_receive('expanduser').with_args('database.kdbx').and_return( + 'database.kdbx' + ) + flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).with_args( + ( + 'keepassxc-cli', + 'show', + '--show-protected', + '--attributes', + 'Password', + 'database.kdbx', + 'mypassword', + '--yubikey', + ) + ).and_return( + 'password' + ).once() + + assert ( + module.load_credential( + hook_config={}, + config={}, + credential_parameters=('database.kdbx', 'mypassword', '--yubikey'), + ) + == 'password' + ) From 863c95414429f29d1959a15b3cd68ed66ace46a9 Mon Sep 17 00:00:00 2001 From: Gautam Aggarwal Date: Sun, 30 Mar 2025 15:57:42 +0000 Subject: [PATCH 200/226] added schema.yaml --- borgmatic/config/schema.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 757820ac..33f0580c 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -2683,6 +2683,16 @@ properties: description: | Command to use instead of "keepassxc-cli". example: /usr/local/bin/keepassxc-cli + key_file: + type: string + description: | + Path to a key file for unlocking the KeePassXC database. + example: /path/to/keyfile + yubikey: + type: boolean + description: | + Whether to use a YubiKey for unlocking the KeePassXC database. + example: true description: | Configuration for integration with the KeePassXC password manager. default_actions: From 92ebc77597fc7504ebfaa1775ef3f603cf8e77e7 Mon Sep 17 00:00:00 2001 From: Gautam Aggarwal Date: Sun, 30 Mar 2025 16:19:56 +0000 Subject: [PATCH 201/226] 2nd Draft --- borgmatic/config/schema.yaml | 2 +- tests/unit/hooks/credential/test_keepassxc.py | 40 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 33f0580c..15c04ee3 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -2683,7 +2683,7 @@ properties: description: | Command to use instead of "keepassxc-cli". example: /usr/local/bin/keepassxc-cli - key_file: + key-file: type: string description: | Path to a key file for unlocking the KeePassXC database. diff --git a/tests/unit/hooks/credential/test_keepassxc.py b/tests/unit/hooks/credential/test_keepassxc.py index 9fddf133..c7c2306f 100644 --- a/tests/unit/hooks/credential/test_keepassxc.py +++ b/tests/unit/hooks/credential/test_keepassxc.py @@ -181,3 +181,43 @@ def test_load_credential_with_yubikey(): ) == 'password' ) + + +def test_load_credential_with_key_file_and_yubikey(): + flexmock(module.os.path).should_receive('expanduser').with_args('database.kdbx').and_return( + 'database.kdbx' + ) + flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).with_args( + ( + 'keepassxc-cli', + 'show', + '--show-protected', + '--attributes', + 'Password', + 'database.kdbx', + 'mypassword', + '--key-file', + '/path/to/keyfile', + '--yubikey', + ) + ).and_return( + 'password' + ).once() + + assert ( + module.load_credential( + hook_config={}, + config={}, + credential_parameters=( + 'database.kdbx', + 'mypassword', + '--key-file', + '/path/to/keyfile', + '--yubikey', + ), + ) + == 'password' + ) From ab01e97a5e728a0e48595c286ddc87312c94d76c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 30 Mar 2025 14:55:54 -0700 Subject: [PATCH 202/226] Fix a "no such file or directory" error in ZFS, Btrfs, and LVM hooks with nested directories that reside on separate devices/filesystems (#1048). --- NEWS | 2 + borgmatic/hooks/data_source/snapshot.py | 8 ++ tests/unit/hooks/data_source/test_snapshot.py | 74 ++++++++++++++++--- 3 files changed, 72 insertions(+), 12 deletions(-) diff --git a/NEWS b/NEWS index 9e5a3e4e..e320b0e8 100644 --- a/NEWS +++ b/NEWS @@ -22,6 +22,8 @@ "working_directory" are used. * #1044: Fix an error in the systemd credential hook when the credential name contains a "." character. + * #1048: Fix a "no such file or directory" error in ZFS, Btrfs, and LVM hooks with nested + directories that reside on separate devices/filesystems. 1.9.14 * #409: With the PagerDuty monitoring hook, send borgmatic logs to PagerDuty so they show up in the diff --git a/borgmatic/hooks/data_source/snapshot.py b/borgmatic/hooks/data_source/snapshot.py index ece63ce6..4b67784f 100644 --- a/borgmatic/hooks/data_source/snapshot.py +++ b/borgmatic/hooks/data_source/snapshot.py @@ -1,3 +1,4 @@ +import os import pathlib IS_A_HOOK = False @@ -11,6 +12,10 @@ def get_contained_patterns(parent_directory, candidate_patterns): paths, but there's a parent directory (logical volume, dataset, subvolume, etc.) at /var, then /var is what we want to snapshot. + If a parent directory and a candidate pattern are on different devices, skip the pattern. That's + because any snapshot of a parent directory won't actually include "contained" directories if + they reside on separate devices. + For this function to work, a candidate pattern path can't have any globs or other non-literal characters in the initial portion of the path that matches the parent directory. For instance, a parent directory of /var would match a candidate pattern path of /var/log/*/data, but not a @@ -27,6 +32,8 @@ def get_contained_patterns(parent_directory, candidate_patterns): if not candidate_patterns: return () + parent_device = os.stat(parent_directory).st_dev + contained_patterns = tuple( candidate for candidate in candidate_patterns @@ -35,6 +42,7 @@ def get_contained_patterns(parent_directory, candidate_patterns): pathlib.PurePath(parent_directory) == candidate_path or pathlib.PurePath(parent_directory) in candidate_path.parents ) + if candidate.device == parent_device ) candidate_patterns -= set(contained_patterns) diff --git a/tests/unit/hooks/data_source/test_snapshot.py b/tests/unit/hooks/data_source/test_snapshot.py index 2c23a1e4..889a5642 100644 --- a/tests/unit/hooks/data_source/test_snapshot.py +++ b/tests/unit/hooks/data_source/test_snapshot.py @@ -1,34 +1,84 @@ +from flexmock import flexmock + from borgmatic.borg.pattern import Pattern from borgmatic.hooks.data_source import snapshot as module def test_get_contained_patterns_without_candidates_returns_empty(): + flexmock(module.os).should_receive('stat').and_return(flexmock(st_dev=flexmock())) + assert module.get_contained_patterns('/mnt', {}) == () def test_get_contained_patterns_with_self_candidate_returns_self(): - candidates = {Pattern('/foo'), Pattern('/mnt'), Pattern('/bar')} + device = flexmock() + flexmock(module.os).should_receive('stat').and_return(flexmock(st_dev=device)) + candidates = { + Pattern('/foo', device=device), + Pattern('/mnt', device=device), + Pattern('/bar', device=device), + } - assert module.get_contained_patterns('/mnt', candidates) == (Pattern('/mnt'),) - assert candidates == {Pattern('/foo'), Pattern('/bar')} + assert module.get_contained_patterns('/mnt', candidates) == (Pattern('/mnt', device=device),) + assert candidates == {Pattern('/foo', device=device), Pattern('/bar', device=device)} def test_get_contained_patterns_with_self_candidate_and_caret_prefix_returns_self(): - candidates = {Pattern('^/foo'), Pattern('^/mnt'), Pattern('^/bar')} + device = flexmock() + flexmock(module.os).should_receive('stat').and_return(flexmock(st_dev=device)) + candidates = { + Pattern('^/foo', device=device), + Pattern('^/mnt', device=device), + Pattern('^/bar', device=device), + } - assert module.get_contained_patterns('/mnt', candidates) == (Pattern('^/mnt'),) - assert candidates == {Pattern('^/foo'), Pattern('^/bar')} + assert module.get_contained_patterns('/mnt', candidates) == (Pattern('^/mnt', device=device),) + assert candidates == {Pattern('^/foo', device=device), Pattern('^/bar', device=device)} def test_get_contained_patterns_with_child_candidate_returns_child(): - candidates = {Pattern('/foo'), Pattern('/mnt/subdir'), Pattern('/bar')} + device = flexmock() + flexmock(module.os).should_receive('stat').and_return(flexmock(st_dev=device)) + candidates = { + Pattern('/foo', device=device), + Pattern('/mnt/subdir', device=device), + Pattern('/bar', device=device), + } - assert module.get_contained_patterns('/mnt', candidates) == (Pattern('/mnt/subdir'),) - assert candidates == {Pattern('/foo'), Pattern('/bar')} + assert module.get_contained_patterns('/mnt', candidates) == ( + Pattern('/mnt/subdir', device=device), + ) + assert candidates == {Pattern('/foo', device=device), Pattern('/bar', device=device)} def test_get_contained_patterns_with_grandchild_candidate_returns_child(): - candidates = {Pattern('/foo'), Pattern('/mnt/sub/dir'), Pattern('/bar')} + device = flexmock() + flexmock(module.os).should_receive('stat').and_return(flexmock(st_dev=device)) + candidates = { + Pattern('/foo', device=device), + Pattern('/mnt/sub/dir', device=device), + Pattern('/bar', device=device), + } - assert module.get_contained_patterns('/mnt', candidates) == (Pattern('/mnt/sub/dir'),) - assert candidates == {Pattern('/foo'), Pattern('/bar')} + assert module.get_contained_patterns('/mnt', candidates) == ( + Pattern('/mnt/sub/dir', device=device), + ) + assert candidates == {Pattern('/foo', device=device), Pattern('/bar', device=device)} + + +def test_get_contained_patterns_ignores_child_candidate_on_another_device(): + one_device = flexmock() + another_device = flexmock() + flexmock(module.os).should_receive('stat').and_return(flexmock(st_dev=one_device)) + candidates = { + Pattern('/foo', device=one_device), + Pattern('/mnt/subdir', device=another_device), + Pattern('/bar', device=one_device), + } + + assert module.get_contained_patterns('/mnt', candidates) == () + assert candidates == { + Pattern('/foo', device=one_device), + Pattern('/mnt/subdir', device=another_device), + Pattern('/bar', device=one_device), + } From 6f07402407928c2da883cd13acbb3fc8bd67aa89 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 30 Mar 2025 19:04:36 -0700 Subject: [PATCH 203/226] Fix end-to-end tests and don't stat() directories that don't exist (#1048). --- borgmatic/hooks/data_source/snapshot.py | 2 +- tests/unit/hooks/data_source/test_snapshot.py | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/borgmatic/hooks/data_source/snapshot.py b/borgmatic/hooks/data_source/snapshot.py index 4b67784f..a89b85df 100644 --- a/borgmatic/hooks/data_source/snapshot.py +++ b/borgmatic/hooks/data_source/snapshot.py @@ -32,7 +32,7 @@ def get_contained_patterns(parent_directory, candidate_patterns): if not candidate_patterns: return () - parent_device = os.stat(parent_directory).st_dev + parent_device = os.stat(parent_directory).st_dev if os.path.exists(parent_directory) else None contained_patterns = tuple( candidate diff --git a/tests/unit/hooks/data_source/test_snapshot.py b/tests/unit/hooks/data_source/test_snapshot.py index 889a5642..8cb98ded 100644 --- a/tests/unit/hooks/data_source/test_snapshot.py +++ b/tests/unit/hooks/data_source/test_snapshot.py @@ -6,6 +6,7 @@ from borgmatic.hooks.data_source import snapshot as module def test_get_contained_patterns_without_candidates_returns_empty(): flexmock(module.os).should_receive('stat').and_return(flexmock(st_dev=flexmock())) + flexmock(module.os.path).should_receive('exists').and_return(True) assert module.get_contained_patterns('/mnt', {}) == () @@ -13,6 +14,7 @@ def test_get_contained_patterns_without_candidates_returns_empty(): def test_get_contained_patterns_with_self_candidate_returns_self(): device = flexmock() flexmock(module.os).should_receive('stat').and_return(flexmock(st_dev=device)) + flexmock(module.os.path).should_receive('exists').and_return(True) candidates = { Pattern('/foo', device=device), Pattern('/mnt', device=device), @@ -26,6 +28,7 @@ def test_get_contained_patterns_with_self_candidate_returns_self(): def test_get_contained_patterns_with_self_candidate_and_caret_prefix_returns_self(): device = flexmock() flexmock(module.os).should_receive('stat').and_return(flexmock(st_dev=device)) + flexmock(module.os.path).should_receive('exists').and_return(True) candidates = { Pattern('^/foo', device=device), Pattern('^/mnt', device=device), @@ -39,6 +42,7 @@ def test_get_contained_patterns_with_self_candidate_and_caret_prefix_returns_sel def test_get_contained_patterns_with_child_candidate_returns_child(): device = flexmock() flexmock(module.os).should_receive('stat').and_return(flexmock(st_dev=device)) + flexmock(module.os.path).should_receive('exists').and_return(True) candidates = { Pattern('/foo', device=device), Pattern('/mnt/subdir', device=device), @@ -54,6 +58,7 @@ def test_get_contained_patterns_with_child_candidate_returns_child(): def test_get_contained_patterns_with_grandchild_candidate_returns_child(): device = flexmock() flexmock(module.os).should_receive('stat').and_return(flexmock(st_dev=device)) + flexmock(module.os.path).should_receive('exists').and_return(True) candidates = { Pattern('/foo', device=device), Pattern('/mnt/sub/dir', device=device), @@ -70,6 +75,7 @@ def test_get_contained_patterns_ignores_child_candidate_on_another_device(): one_device = flexmock() another_device = flexmock() flexmock(module.os).should_receive('stat').and_return(flexmock(st_dev=one_device)) + flexmock(module.os.path).should_receive('exists').and_return(True) candidates = { Pattern('/foo', device=one_device), Pattern('/mnt/subdir', device=another_device), @@ -82,3 +88,21 @@ def test_get_contained_patterns_ignores_child_candidate_on_another_device(): Pattern('/mnt/subdir', device=another_device), Pattern('/bar', device=one_device), } + + +def test_get_contained_patterns_with_non_existent_parent_directory_ignores_child_candidate(): + device = flexmock() + flexmock(module.os).should_receive('stat').and_return(flexmock(st_dev=device)) + flexmock(module.os.path).should_receive('exists').and_return(False) + candidates = { + Pattern('/foo', device=device), + Pattern('/mnt/subdir', device=device), + Pattern('/bar', device=device), + } + + assert module.get_contained_patterns('/mnt', candidates) == () + assert candidates == { + Pattern('/foo', device=device), + Pattern('/mnt/subdir', device=device), + Pattern('/bar', device=device), + } From cf477bdc1c65e6e7210a7ee6049b1752f47f6806 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 31 Mar 2025 11:33:56 -0700 Subject: [PATCH 204/226] Fix broken list_details, progress, and statistics options (#303). --- borgmatic/actions/check.py | 3 +- borgmatic/actions/compact.py | 10 - borgmatic/actions/config/bootstrap.py | 1 - borgmatic/actions/create.py | 25 +-- borgmatic/actions/export_tar.py | 5 - borgmatic/actions/extract.py | 5 - borgmatic/actions/transfer.py | 6 + borgmatic/borg/check.py | 11 +- borgmatic/borg/compact.py | 5 +- borgmatic/borg/create.py | 17 +- borgmatic/borg/delete.py | 21 +- borgmatic/borg/export_tar.py | 5 +- borgmatic/borg/extract.py | 9 +- borgmatic/borg/info.py | 4 +- borgmatic/borg/prune.py | 16 +- borgmatic/borg/recreate.py | 9 +- borgmatic/borg/repo_delete.py | 4 +- borgmatic/borg/repo_list.py | 2 +- borgmatic/borg/transfer.py | 11 +- borgmatic/commands/arguments.py | 79 ++++---- borgmatic/commands/borgmatic.py | 18 +- borgmatic/config/arguments.py | 13 +- borgmatic/config/validate.py | 12 +- tests/integration/commands/test_arguments.py | 38 +--- tests/integration/config/test_validate.py | 24 +-- tests/unit/actions/config/test_bootstrap.py | 3 - tests/unit/actions/test_check.py | 7 +- tests/unit/actions/test_compact.py | 93 ++------- tests/unit/actions/test_create.py | 194 +++++++------------ tests/unit/actions/test_export_tar.py | 8 +- tests/unit/actions/test_extract.py | 2 - tests/unit/actions/test_prune.py | 10 +- tests/unit/actions/test_transfer.py | 21 +- tests/unit/borg/test_check.py | 53 +---- tests/unit/borg/test_compact.py | 26 ++- tests/unit/borg/test_create.py | 28 ++- tests/unit/borg/test_delete.py | 52 ++--- tests/unit/borg/test_export_tar.py | 15 +- tests/unit/borg/test_extract.py | 8 +- tests/unit/borg/test_info.py | 2 +- tests/unit/borg/test_prune.py | 108 ++--------- tests/unit/borg/test_recreate.py | 32 +-- tests/unit/borg/test_repo_delete.py | 26 +-- tests/unit/borg/test_repo_list.py | 2 +- tests/unit/borg/test_transfer.py | 35 +--- tests/unit/commands/test_borgmatic.py | 6 +- tests/unit/config/test_arguments.py | 12 ++ 47 files changed, 357 insertions(+), 739 deletions(-) diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index b562d760..afcc9262 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -372,7 +372,7 @@ def collect_spot_check_source_paths( borgmatic.borg.create.make_base_create_command( dry_run=True, repository_path=repository['path'], - config=config, + config=dict(config, list_details=True), patterns=borgmatic.actions.create.process_patterns( borgmatic.actions.create.collect_patterns(config), working_directory, @@ -382,7 +382,6 @@ def collect_spot_check_source_paths( borgmatic_runtime_directory=borgmatic_runtime_directory, local_path=local_path, remote_path=remote_path, - list_files=True, stream_processes=stream_processes, ) ) diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index dbc2e573..551c680d 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -37,17 +37,7 @@ def run_compact( global_arguments, local_path=local_path, remote_path=remote_path, - progress=( - config.get('progress') - if compact_arguments.progress is None - else compact_arguments.progress - ), cleanup_commits=compact_arguments.cleanup_commits, - threshold=( - config.get('compact_threshold') - if compact_arguments.threshold is None - else compact_arguments.threshold - ), ) else: # pragma: nocover logger.info('Skipping compact (only available/needed in Borg 1.2+)') diff --git a/borgmatic/actions/config/bootstrap.py b/borgmatic/actions/config/bootstrap.py index 4a72b2cd..74714c7e 100644 --- a/borgmatic/actions/config/bootstrap.py +++ b/borgmatic/actions/config/bootstrap.py @@ -127,5 +127,4 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version): extract_to_stdout=False, destination_path=bootstrap_arguments.destination, strip_components=bootstrap_arguments.strip_components, - progress=bootstrap_arguments.progress, ) diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index e0ef3b1f..a92af261 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -289,6 +289,16 @@ def run_create( ): return + if config.get('list_details') and config.get('progress'): + raise ValueError( + 'With the create action, only one of --list/--files/list_details and --progress/progress can be used.' + ) + + if config.get('list_details') and create_arguments.json: + raise ValueError( + 'With the create action, only one of --list/--files/list_details and --json can be used.' + ) + logger.info(f'Creating archive{dry_run_label}') working_directory = borgmatic.config.paths.get_working_directory(config) @@ -327,22 +337,7 @@ def run_create( borgmatic_runtime_directory, local_path=local_path, remote_path=remote_path, - progress=( - config.get('progress') - if create_arguments.progress is None - else create_arguments.progress - ), - stats=( - config.get('statistics') - if create_arguments.stats is None - else create_arguments.stats - ), json=create_arguments.json, - list_files=( - config.get('list_details') - if create_arguments.list_files is None - else create_arguments.list_files - ), stream_processes=stream_processes, ) diff --git a/borgmatic/actions/export_tar.py b/borgmatic/actions/export_tar.py index cc18aeeb..f04b06ff 100644 --- a/borgmatic/actions/export_tar.py +++ b/borgmatic/actions/export_tar.py @@ -43,10 +43,5 @@ def run_export_tar( local_path=local_path, remote_path=remote_path, tar_filter=export_tar_arguments.tar_filter, - list_files=( - config.get('list_details') - if export_tar_arguments.list_files is None - else export_tar_arguments.list_files - ), strip_components=export_tar_arguments.strip_components, ) diff --git a/borgmatic/actions/extract.py b/borgmatic/actions/extract.py index 3e42b57d..feaaac81 100644 --- a/borgmatic/actions/extract.py +++ b/borgmatic/actions/extract.py @@ -45,9 +45,4 @@ def run_extract( remote_path=remote_path, destination_path=extract_arguments.destination, strip_components=extract_arguments.strip_components, - progress=( - config.get('progress') - if extract_arguments.progress is None - else extract_arguments.progress - ), ) diff --git a/borgmatic/actions/transfer.py b/borgmatic/actions/transfer.py index 8a27d16b..c9601515 100644 --- a/borgmatic/actions/transfer.py +++ b/borgmatic/actions/transfer.py @@ -17,7 +17,13 @@ def run_transfer( ''' Run the "transfer" action for the given repository. ''' + if transfer_arguments.archive and config.get('match_archives'): + raise ValueError( + 'With the transfer action, only one of --archive and --match-archives/match_archives can be used.' + ) + logger.info('Transferring archives to repository') + borgmatic.borg.transfer.transfer_archives( global_arguments.dry_run, repository['path'], diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index c02b6ec4..935c52c8 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -32,7 +32,7 @@ def make_archive_filter_flags(local_borg_version, config, checks, check_argument if prefix else ( flags.make_match_archives_flags( - check_arguments.match_archives or config.get('match_archives'), + config.get('match_archives'), config.get('archive_name_format'), local_borg_version, ) @@ -143,9 +143,6 @@ def check_archives( umask = config.get('umask') borg_exit_codes = config.get('borg_exit_codes') working_directory = borgmatic.config.paths.get_working_directory(config) - progress = ( - config.get('progress') if check_arguments.progress is None else check_arguments.progress - ) if 'data' in checks: checks.add('archives') @@ -173,7 +170,7 @@ def check_archives( + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + verbosity_flags - + (('--progress',) if progress else ()) + + (('--progress',) if config.get('progress') else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + flags.make_repository_flags(repository_path, local_borg_version) ) @@ -182,7 +179,9 @@ def check_archives( full_command, # The Borg repair option triggers an interactive prompt, which won't work when output is # captured. And progress messes with the terminal directly. - output_file=(DO_NOT_CAPTURE if check_arguments.repair or progress else None), + output_file=( + DO_NOT_CAPTURE if check_arguments.repair or config.get('progress') else None + ), environment=environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, diff --git a/borgmatic/borg/compact.py b/borgmatic/borg/compact.py index fd248cbf..b443e8c0 100644 --- a/borgmatic/borg/compact.py +++ b/borgmatic/borg/compact.py @@ -15,9 +15,7 @@ def compact_segments( global_arguments, local_path='borg', remote_path=None, - progress=False, cleanup_commits=False, - threshold=None, ): ''' Given dry-run flag, a local or remote repository path, a configuration dict, and the local Borg @@ -26,6 +24,7 @@ def compact_segments( umask = config.get('umask', None) lock_wait = config.get('lock_wait', None) extra_borg_options = config.get('extra_borg_options', {}).get('compact', '') + threshold = config.get('compact_threshold') full_command = ( (local_path, 'compact') @@ -33,7 +32,7 @@ def compact_segments( + (('--umask', str(umask)) if umask else ()) + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) - + (('--progress',) if progress else ()) + + (('--progress',) if config.get('progress') else ()) + (('--cleanup-commits',) if cleanup_commits else ()) + (('--threshold', str(threshold)) if threshold else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index ccdb41a9..de5115dc 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -213,9 +213,7 @@ def make_base_create_command( borgmatic_runtime_directory, local_path='borg', remote_path=None, - progress=False, json=False, - list_files=False, stream_processes=None, ): ''' @@ -293,7 +291,7 @@ def make_base_create_command( + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + ( ('--list', '--filter', list_filter_flags) - if list_files and not json and not progress + if config.get('list_details') and not json and not config.get('progress') else () ) + (('--dry-run',) if dry_run else ()) @@ -361,10 +359,7 @@ def create_archive( borgmatic_runtime_directory, local_path='borg', remote_path=None, - progress=False, - stats=False, json=False, - list_files=False, stream_processes=None, ): ''' @@ -389,28 +384,26 @@ def create_archive( borgmatic_runtime_directory, local_path, remote_path, - progress, json, - list_files, stream_processes, ) if json: output_log_level = None - elif list_files or (stats and not dry_run): + elif config.get('list_details') or (config.get('statistics') and not dry_run): output_log_level = logging.ANSWER else: output_log_level = logging.INFO # The progress output isn't compatible with captured and logged output, as progress messes with # the terminal directly. - output_file = DO_NOT_CAPTURE if progress else None + output_file = DO_NOT_CAPTURE if config.get('progress') else None create_flags += ( (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ()) - + (('--stats',) if stats and not json and not dry_run else ()) + + (('--stats',) if config.get('statistics') and not json and not dry_run else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else ()) - + (('--progress',) if progress else ()) + + (('--progress',) if config.get('progress') else ()) + (('--json',) if json else ()) ) borg_exit_codes = config.get('borg_exit_codes') diff --git a/borgmatic/borg/delete.py b/borgmatic/borg/delete.py index f31730ce..63ee0ef8 100644 --- a/borgmatic/borg/delete.py +++ b/borgmatic/borg/delete.py @@ -34,14 +34,7 @@ def make_delete_command( + borgmatic.borg.flags.make_flags('umask', config.get('umask')) + borgmatic.borg.flags.make_flags('log-json', global_arguments.log_json) + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait')) - + borgmatic.borg.flags.make_flags( - 'list', - ( - config.get('list_details') - if delete_arguments.list_archives is None - else delete_arguments.list_archives - ), - ) + + borgmatic.borg.flags.make_flags('list', config.get('list_details')) + ( (('--force',) + (('--force',) if delete_arguments.force >= 2 else ())) if delete_arguments.force @@ -55,9 +48,17 @@ def make_delete_command( local_borg_version=local_borg_version, default_archive_name_format='*', ) + + (('--stats',) if config.get('statistics') else ()) + borgmatic.borg.flags.make_flags_from_arguments( delete_arguments, - excludes=('list_archives', 'force', 'match_archives', 'archive', 'repository'), + excludes=( + 'list_details', + 'statistics', + 'force', + 'match_archives', + 'archive', + 'repository', + ), ) + borgmatic.borg.flags.make_repository_flags(repository['path'], local_borg_version) ) @@ -105,7 +106,7 @@ def delete_archives( repo_delete_arguments = argparse.Namespace( repository=repository['path'], - list_archives=delete_arguments.list_archives, + list_details=delete_arguments.list_details, force=delete_arguments.force, cache_only=delete_arguments.cache_only, keep_security_info=delete_arguments.keep_security_info, diff --git a/borgmatic/borg/export_tar.py b/borgmatic/borg/export_tar.py index d8283535..224c07b1 100644 --- a/borgmatic/borg/export_tar.py +++ b/borgmatic/borg/export_tar.py @@ -20,7 +20,6 @@ def export_tar_archive( local_path='borg', remote_path=None, tar_filter=None, - list_files=False, strip_components=None, ): ''' @@ -43,7 +42,7 @@ def export_tar_archive( + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) - + (('--list',) if list_files else ()) + + (('--list',) if config.get('list_details') else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (('--dry-run',) if dry_run else ()) + (('--tar-filter', tar_filter) if tar_filter else ()) @@ -57,7 +56,7 @@ def export_tar_archive( + (tuple(paths) if paths else ()) ) - if list_files: + if config.get('list_details'): output_log_level = logging.ANSWER else: output_log_level = logging.INFO diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index 52d7c35f..aa354cbc 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -77,7 +77,6 @@ def extract_archive( remote_path=None, destination_path=None, strip_components=None, - progress=False, extract_to_stdout=False, ): ''' @@ -92,8 +91,8 @@ def extract_archive( umask = config.get('umask', None) lock_wait = config.get('lock_wait', None) - if progress and extract_to_stdout: - raise ValueError('progress and extract_to_stdout cannot both be set') + if config.get('progress') and extract_to_stdout: + raise ValueError('progress and extract to stdout cannot both be set') if feature.available(feature.Feature.NUMERIC_IDS, local_borg_version): numeric_ids_flags = ('--numeric-ids',) if config.get('numeric_ids') else () @@ -128,7 +127,7 @@ def extract_archive( + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (('--dry-run',) if dry_run else ()) + (('--strip-components', str(strip_components)) if strip_components else ()) - + (('--progress',) if progress else ()) + + (('--progress',) if config.get('progress') else ()) + (('--stdout',) if extract_to_stdout else ()) + flags.make_repository_archive_flags( # Make the repository path absolute so the destination directory used below via changing @@ -148,7 +147,7 @@ def extract_archive( # The progress output isn't compatible with captured and logged output, as progress messes with # the terminal directly. - if progress: + if config.get('progress'): return execute_command( full_command, output_file=DO_NOT_CAPTURE, diff --git a/borgmatic/borg/info.py b/borgmatic/borg/info.py index bde80a97..2d90ef52 100644 --- a/borgmatic/borg/info.py +++ b/borgmatic/borg/info.py @@ -48,9 +48,7 @@ def make_info_command( if info_arguments.prefix else ( flags.make_match_archives_flags( - info_arguments.match_archives - or info_arguments.archive - or config.get('match_archives'), + info_arguments.archive or config.get('match_archives'), config.get('archive_name_format'), local_borg_version, ) diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index b29cbe67..34530eeb 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -41,7 +41,7 @@ def make_prune_flags(config, prune_arguments, local_borg_version): if prefix else ( flags.make_match_archives_flags( - prune_arguments.match_archives or config.get('match_archives'), + config.get('match_archives'), config.get('archive_name_format'), local_borg_version, ) @@ -66,12 +66,6 @@ def prune_archives( borgmatic.logger.add_custom_log_levels() umask = config.get('umask', None) lock_wait = config.get('lock_wait', None) - stats = config.get('statistics') if prune_arguments.stats is None else prune_arguments.stats - list_archives = ( - config.get('list_details') - if prune_arguments.list_archives is None - else prune_arguments.list_archives - ) extra_borg_options = config.get('extra_borg_options', {}).get('prune', '') full_command = ( @@ -83,7 +77,7 @@ def prune_archives( + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + ( ('--stats',) - if stats + if config.get('statistics') and not dry_run and not feature.available(feature.Feature.NO_PRUNE_STATS, local_borg_version) else () @@ -91,16 +85,16 @@ def prune_archives( + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + flags.make_flags_from_arguments( prune_arguments, - excludes=('repository', 'match_archives', 'stats', 'list_archives'), + excludes=('repository', 'match_archives', 'statistics', 'list_details'), ) - + (('--list',) if list_archives else ()) + + (('--list',) if config.get('list_details') else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (('--dry-run',) if dry_run else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + flags.make_repository_flags(repository_path, local_borg_version) ) - if stats or list_archives: + if config.get('statistics') or config.get('list_details'): output_log_level = logging.ANSWER else: output_log_level = logging.INFO diff --git a/borgmatic/borg/recreate.py b/borgmatic/borg/recreate.py index 15969f5f..e6f1e04d 100644 --- a/borgmatic/borg/recreate.py +++ b/borgmatic/borg/recreate.py @@ -38,9 +38,6 @@ def recreate_archive( patterns_file = write_patterns_file( patterns, borgmatic.config.paths.get_working_directory(config) ) - list_files = ( - config.get('list_details') if recreate_arguments.list is None else recreate_arguments.list - ) recreate_command = ( (local_path, 'recreate') @@ -56,7 +53,7 @@ def recreate_archive( '--filter', make_list_filter_flags(local_borg_version, global_arguments.dry_run), ) - if list_files + if config.get('list_details') else () ) # Flag --target works only for a single archive. @@ -71,12 +68,10 @@ def recreate_archive( + (('--chunker-params', chunker_params) if chunker_params else ()) + ( flags.make_match_archives_flags( - recreate_arguments.match_archives or archive or config.get('match_archives'), + archive or config.get('match_archives'), config.get('archive_name_format'), local_borg_version, ) - if recreate_arguments.match_archives - else () ) + (('--recompress', recompress) if recompress else ()) + exclude_flags diff --git a/borgmatic/borg/repo_delete.py b/borgmatic/borg/repo_delete.py index fa66d3c0..dee5a1a9 100644 --- a/borgmatic/borg/repo_delete.py +++ b/borgmatic/borg/repo_delete.py @@ -39,14 +39,14 @@ def make_repo_delete_command( + borgmatic.borg.flags.make_flags('umask', config.get('umask')) + borgmatic.borg.flags.make_flags('log-json', global_arguments.log_json) + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait')) - + borgmatic.borg.flags.make_flags('list', repo_delete_arguments.list_archives) + + borgmatic.borg.flags.make_flags('list', config.get('list_details')) + ( (('--force',) + (('--force',) if repo_delete_arguments.force >= 2 else ())) if repo_delete_arguments.force else () ) + borgmatic.borg.flags.make_flags_from_arguments( - repo_delete_arguments, excludes=('list_archives', 'force', 'repository') + repo_delete_arguments, excludes=('list_details', 'force', 'repository') ) + borgmatic.borg.flags.make_repository_flags(repository['path'], local_borg_version) ) diff --git a/borgmatic/borg/repo_list.py b/borgmatic/borg/repo_list.py index 9722758f..9f05adbf 100644 --- a/borgmatic/borg/repo_list.py +++ b/borgmatic/borg/repo_list.py @@ -113,7 +113,7 @@ def make_repo_list_command( if repo_list_arguments.prefix else ( flags.make_match_archives_flags( - repo_list_arguments.match_archives or config.get('match_archives'), + config.get('match_archives'), config.get('archive_name_format'), local_borg_version, ) diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index 528cd64c..5e16efce 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -40,9 +40,7 @@ def transfer_archives( ) or ( flags.make_match_archives_flags( - transfer_arguments.match_archives - or transfer_arguments.archive - or config.get('match_archives'), + transfer_arguments.archive or config.get('match_archives'), config.get('archive_name_format'), local_borg_version, ) @@ -52,16 +50,11 @@ def transfer_archives( + flags.make_flags('other-repo', transfer_arguments.source_repository) + flags.make_flags('dry-run', dry_run) ) - progress = ( - config.get('progress') - if transfer_arguments.progress is None - else transfer_arguments.progress - ) return execute_command( full_command, output_log_level=logging.ANSWER, - output_file=DO_NOT_CAPTURE if progress else None, + output_file=DO_NOT_CAPTURE if config.get('progress') else None, environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index b4623a10..c3b61f42 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -709,7 +709,7 @@ def make_parsers(schema, unparsed_arguments): ) transfer_group.add_argument( '--progress', - default=False, + default=None, action='store_true', help='Display progress as each archive is transferred', ) @@ -776,13 +776,17 @@ def make_parsers(schema, unparsed_arguments): ) prune_group.add_argument( '--stats', - dest='stats', - default=False, + dest='statistics', + default=None, action='store_true', help='Display statistics of the pruned archive [Borg 1 only]', ) prune_group.add_argument( - '--list', dest='list_archives', action='store_true', help='List archives kept/pruned' + '--list', + dest='list_details', + default=None, + action='store_true', + help='List archives kept/pruned', ) prune_group.add_argument( '--oldest', @@ -820,8 +824,7 @@ def make_parsers(schema, unparsed_arguments): ) compact_group.add_argument( '--progress', - dest='progress', - default=False, + default=None, action='store_true', help='Display progress as each segment is compacted', ) @@ -835,7 +838,7 @@ def make_parsers(schema, unparsed_arguments): compact_group.add_argument( '--threshold', type=int, - dest='threshold', + dest='compact_threshold', help='Minimum saved space percentage threshold for compacting a segment, defaults to 10', ) compact_group.add_argument( @@ -856,20 +859,24 @@ def make_parsers(schema, unparsed_arguments): ) create_group.add_argument( '--progress', - dest='progress', - default=False, + default=None, action='store_true', help='Display progress for each file as it is backed up', ) create_group.add_argument( '--stats', - dest='stats', - default=False, + dest='statistics', + default=None, action='store_true', help='Display statistics of archive', ) create_group.add_argument( - '--list', '--files', dest='list_files', action='store_true', help='Show per-file details' + '--list', + '--files', + dest='list_details', + default=None, + action='store_true', + help='Show per-file details', ) create_group.add_argument( '--json', dest='json', default=False, action='store_true', help='Output results as JSON' @@ -890,8 +897,7 @@ def make_parsers(schema, unparsed_arguments): ) check_group.add_argument( '--progress', - dest='progress', - default=False, + default=None, action='store_true', help='Display progress for each file as it is checked', ) @@ -948,12 +954,15 @@ def make_parsers(schema, unparsed_arguments): ) delete_group.add_argument( '--list', - dest='list_archives', + dest='list_details', + default=None, action='store_true', help='Show details for the deleted archives', ) delete_group.add_argument( '--stats', + dest='statistics', + default=None, action='store_true', help='Display statistics for the deleted archives', ) @@ -1058,8 +1067,7 @@ def make_parsers(schema, unparsed_arguments): ) extract_group.add_argument( '--progress', - dest='progress', - default=False, + default=None, action='store_true', help='Display progress for each file as it is extracted', ) @@ -1134,8 +1142,7 @@ def make_parsers(schema, unparsed_arguments): ) config_bootstrap_group.add_argument( '--progress', - dest='progress', - default=False, + default=None, action='store_true', help='Display progress for each file as it is extracted', ) @@ -1228,7 +1235,12 @@ def make_parsers(schema, unparsed_arguments): '--tar-filter', help='Name of filter program to pipe data through' ) export_tar_group.add_argument( - '--list', '--files', dest='list_files', action='store_true', help='Show per-file details' + '--list', + '--files', + dest='list_details', + default=None, + action='store_true', + help='Show per-file details', ) export_tar_group.add_argument( '--strip-components', @@ -1339,7 +1351,8 @@ def make_parsers(schema, unparsed_arguments): ) repo_delete_group.add_argument( '--list', - dest='list_archives', + dest='list_details', + default=None, action='store_true', help='Show details for the archives in the given repository', ) @@ -1770,7 +1783,11 @@ def make_parsers(schema, unparsed_arguments): help='Archive name, hash, or series to recreate', ) recreate_group.add_argument( - '--list', dest='list', action='store_true', help='Show per-file details' + '--list', + dest='list_details', + default=None, + action='store_true', + help='Show per-file details', ) recreate_group.add_argument( '--target', @@ -1865,15 +1882,6 @@ def parse_arguments(schema, *unparsed_arguments): f"Unrecognized argument{'s' if len(unknown_arguments) > 1 else ''}: {' '.join(unknown_arguments)}" ) - if 'create' in arguments and arguments['create'].list_files and arguments['create'].progress: - raise ValueError( - 'With the create action, only one of --list (--files) and --progress flags can be used.' - ) - if 'create' in arguments and arguments['create'].list_files and arguments['create'].json: - raise ValueError( - 'With the create action, only one of --list (--files) and --json flags can be used.' - ) - if ( ('list' in arguments and 'repo-info' in arguments and arguments['list'].json) or ('list' in arguments and 'info' in arguments and arguments['list'].json) @@ -1881,15 +1889,6 @@ def parse_arguments(schema, *unparsed_arguments): ): raise ValueError('With the --json flag, multiple actions cannot be used together.') - if ( - 'transfer' in arguments - and arguments['transfer'].archive - and arguments['transfer'].match_archives - ): - raise ValueError( - 'With the transfer action, only one of --archive and --match-archives flags can be used.' - ) - if 'list' in arguments and (arguments['list'].prefix and arguments['list'].match_archives): raise ValueError( 'With the list action, only one of --prefix or --match-archives flags can be used.' diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index c457c8be..28a92b53 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -600,14 +600,14 @@ def run_actions( ) -def load_configurations(config_filenames, global_arguments, overrides=None, resolve_env=True): +def load_configurations(config_filenames, arguments, overrides=None, resolve_env=True): ''' - Given a sequence of configuration filenames, global arguments as an argparse.Namespace, a - sequence of configuration file override strings in the form of "option.suboption=value", and - whether to resolve environment variables, load and validate each configuration file. Return the - results as a tuple of: dict of configuration filename to corresponding parsed configuration, a - sequence of paths for all loaded configuration files (including includes), and a sequence of - logging.LogRecord instances containing any parse errors. + Given a sequence of configuration filenames, arguments as a dict from action name to + argparse.Namespace, a sequence of configuration file override strings in the form of + "option.suboption=value", and whether to resolve environment variables, load and validate each + configuration file. Return the results as a tuple of: dict of configuration filename to + corresponding parsed configuration, a sequence of paths for all loaded configuration files + (including includes), and a sequence of logging.LogRecord instances containing any parse errors. Log records are returned here instead of being logged directly because logging isn't yet initialized at this point! (Although with the Delayed_logging_handler now in place, maybe this @@ -635,7 +635,7 @@ def load_configurations(config_filenames, global_arguments, overrides=None, reso configs[config_filename], paths, parse_logs = validate.parse_configuration( config_filename, validate.schema_filename(), - global_arguments, + arguments, overrides, resolve_env, ) @@ -1010,7 +1010,7 @@ def main(extra_summary_logs=[]): # pragma: no cover config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths)) configs, config_paths, parse_logs = load_configurations( config_filenames, - global_arguments, + arguments, global_arguments.overrides, resolve_env=global_arguments.resolve_env and not validate, ) diff --git a/borgmatic/config/arguments.py b/borgmatic/config/arguments.py index 85c8eef9..3252d779 100644 --- a/borgmatic/config/arguments.py +++ b/borgmatic/config/arguments.py @@ -160,15 +160,16 @@ def prepare_arguments_for_config(global_arguments, schema): return tuple(prepared_values) -def apply_arguments_to_config(config, schema, global_arguments): # pragma: no cover +def apply_arguments_to_config(config, schema, arguments): ''' - Given a configuration dict, a corresponding configuration schema dict, and global arguments as - an argparse.Namespace, set those given argument values into their corresponding configuration - options in the configuration dict. + Given a configuration dict, a corresponding configuration schema dict, and arguments as a dict + from action name to argparse.Namespace, set those given argument values into their corresponding + configuration options in the configuration dict. This supports argument flags of the from "--foo.bar.baz" where each dotted component is a nested configuration object. Additionally, flags like "--foo.bar[0].baz" are supported to update a list element in the configuration. ''' - for keys, value in prepare_arguments_for_config(global_arguments, schema): - set_values(config, keys, value) + for action_arguments in arguments.values(): + for keys, value in prepare_arguments_for_config(action_arguments, schema): + set_values(config, keys, value) diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index 98bf080c..9cb02243 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -97,14 +97,14 @@ def apply_logical_validation(config_filename, parsed_configuration): def parse_configuration( - config_filename, schema_filename, global_arguments, overrides=None, resolve_env=True + config_filename, schema_filename, arguments, overrides=None, resolve_env=True ): ''' Given the path to a config filename in YAML format, the path to a schema filename in a YAML - rendition of JSON Schema format, global arguments as an argparse.Namespace, a sequence of - configuration file override strings in the form of "option.suboption=value", and whether to - resolve environment variables, return the parsed configuration as a data structure of nested - dicts and lists corresponding to the schema. Example return value. + rendition of JSON Schema format, arguments as dict from action name to argparse.Namespace, a + sequence of configuration file override strings in the form of "option.suboption=value", and + whether to resolve environment variables, return the parsed configuration as a data structure of + nested dicts and lists corresponding to the schema. Example return value. Example return value: @@ -129,7 +129,7 @@ def parse_configuration( except (ruamel.yaml.error.YAMLError, RecursionError) as error: raise Validation_error(config_filename, (str(error),)) - borgmatic.config.arguments.apply_arguments_to_config(config, schema, global_arguments) + borgmatic.config.arguments.apply_arguments_to_config(config, schema, arguments) override.apply_overrides(config, schema, overrides) constants.apply_constants(config, config.get('constants') if config else {}) diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index ed6a9892..a357e1b0 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -271,11 +271,11 @@ def test_parse_arguments_with_no_actions_passes_argument_to_relevant_actions(): arguments = module.parse_arguments({}, '--stats', '--list') assert 'prune' in arguments - assert arguments['prune'].stats - assert arguments['prune'].list_archives + assert arguments['prune'].statistics + assert arguments['prune'].list_details assert 'create' in arguments - assert arguments['create'].stats - assert arguments['create'].list_files + assert arguments['create'].statistics + assert arguments['create'].list_details assert 'check' in arguments @@ -557,20 +557,6 @@ def test_parse_arguments_with_list_flag_but_no_relevant_action_raises_value_erro module.parse_arguments({}, '--list', 'repo-create') -def test_parse_arguments_disallows_list_with_progress_for_create_action(): - flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - - with pytest.raises(ValueError): - module.parse_arguments({}, 'create', '--list', '--progress') - - -def test_parse_arguments_disallows_list_with_json_for_create_action(): - flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - - with pytest.raises(ValueError): - module.parse_arguments({}, 'create', '--list', '--json') - - def test_parse_arguments_allows_json_with_list_or_info(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) @@ -599,22 +585,6 @@ def test_parse_arguments_disallows_json_with_both_repo_info_and_info(): module.parse_arguments({}, 'repo-info', 'info', '--json') -def test_parse_arguments_disallows_transfer_with_both_archive_and_match_archives(): - flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - - with pytest.raises(ValueError): - module.parse_arguments( - {}, - 'transfer', - '--source-repository', - 'source.borg', - '--archive', - 'foo', - '--match-archives', - 'sh:*bar', - ) - - def test_parse_arguments_disallows_list_with_both_prefix_and_match_archives(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) diff --git a/tests/integration/config/test_validate.py b/tests/integration/config/test_validate.py index 61d12d7f..e74989c6 100644 --- a/tests/integration/config/test_validate.py +++ b/tests/integration/config/test_validate.py @@ -59,7 +59,7 @@ def test_parse_configuration_transforms_file_into_mapping(): ) config, config_paths, logs = module.parse_configuration( - '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} ) assert config == { @@ -89,7 +89,7 @@ def test_parse_configuration_passes_through_quoted_punctuation(): ) config, config_paths, logs = module.parse_configuration( - '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} ) assert config == { @@ -123,7 +123,9 @@ def test_parse_configuration_with_schema_lacking_examples_does_not_raise(): ''', ) - module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock()) + module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} + ) def test_parse_configuration_inlines_include_inside_deprecated_section(): @@ -150,7 +152,7 @@ def test_parse_configuration_inlines_include_inside_deprecated_section(): builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) config, config_paths, logs = module.parse_configuration( - '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} ) assert config == { @@ -188,7 +190,7 @@ def test_parse_configuration_merges_include(): builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) config, config_paths, logs = module.parse_configuration( - '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} ) assert config == { @@ -205,7 +207,7 @@ def test_parse_configuration_merges_include(): def test_parse_configuration_raises_for_missing_config_file(): with pytest.raises(FileNotFoundError): module.parse_configuration( - '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} ) @@ -219,7 +221,7 @@ def test_parse_configuration_raises_for_missing_schema_file(): with pytest.raises(FileNotFoundError): module.parse_configuration( - '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} ) @@ -228,7 +230,7 @@ def test_parse_configuration_raises_for_syntax_error(): with pytest.raises(ValueError): module.parse_configuration( - '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} ) @@ -243,7 +245,7 @@ def test_parse_configuration_raises_for_validation_error(): with pytest.raises(module.Validation_error): module.parse_configuration( - '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} ) @@ -263,7 +265,7 @@ def test_parse_configuration_applies_overrides(): config, config_paths, logs = module.parse_configuration( '/tmp/config.yaml', '/tmp/schema.yaml', - global_arguments=flexmock(), + arguments={'global': flexmock()}, overrides=['local_path=borg2'], ) @@ -293,7 +295,7 @@ def test_parse_configuration_applies_normalization_after_environment_variable_in flexmock(os).should_receive('getenv').replace_with(lambda variable_name, default: default) config, config_paths, logs = module.parse_configuration( - '/tmp/config.yaml', '/tmp/schema.yaml', global_arguments=flexmock() + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} ) assert config == { diff --git a/tests/unit/actions/config/test_bootstrap.py b/tests/unit/actions/config/test_bootstrap.py index 7a21e715..554ad05d 100644 --- a/tests/unit/actions/config/test_bootstrap.py +++ b/tests/unit/actions/config/test_bootstrap.py @@ -267,7 +267,6 @@ def test_run_bootstrap_does_not_raise(): archive='archive', destination='dest', strip_components=1, - progress=False, user_runtime_directory='/borgmatic', ssh_command=None, local_path='borg7', @@ -307,7 +306,6 @@ def test_run_bootstrap_translates_ssh_command_argument_to_config(): archive='archive', destination='dest', strip_components=1, - progress=False, user_runtime_directory='/borgmatic', ssh_command='ssh -i key', local_path='borg7', @@ -339,7 +337,6 @@ def test_run_bootstrap_translates_ssh_command_argument_to_config(): extract_to_stdout=False, destination_path='dest', strip_components=1, - progress=False, local_path='borg7', remote_path='borg8', ).and_return(extract_process).once() diff --git a/tests/unit/actions/test_check.py b/tests/unit/actions/test_check.py index 6b16a901..069fae1e 100644 --- a/tests/unit/actions/test_check.py +++ b/tests/unit/actions/test_check.py @@ -577,7 +577,6 @@ def test_collect_spot_check_source_paths_parses_borg_output(): borgmatic_runtime_directory='/run/borgmatic', local_path=object, remote_path=object, - list_files=True, stream_processes=True, ).and_return((('borg', 'create'), ('repo::archive',), flexmock())) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -625,7 +624,6 @@ def test_collect_spot_check_source_paths_passes_through_stream_processes_false() borgmatic_runtime_directory='/run/borgmatic', local_path=object, remote_path=object, - list_files=True, stream_processes=False, ).and_return((('borg', 'create'), ('repo::archive',), flexmock())) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -673,7 +671,6 @@ def test_collect_spot_check_source_paths_without_working_directory_parses_borg_o borgmatic_runtime_directory='/run/borgmatic', local_path=object, remote_path=object, - list_files=True, stream_processes=True, ).and_return((('borg', 'create'), ('repo::archive',), flexmock())) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -721,7 +718,6 @@ def test_collect_spot_check_source_paths_skips_directories(): borgmatic_runtime_directory='/run/borgmatic', local_path=object, remote_path=object, - list_files=True, stream_processes=True, ).and_return((('borg', 'create'), ('repo::archive',), flexmock())) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -860,14 +856,13 @@ def test_collect_spot_check_source_paths_uses_working_directory(): flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args( dry_run=True, repository_path='repo', - config=object, + config={'working_directory': '/working/dir', 'list_details': True}, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version=object, global_arguments=object, borgmatic_runtime_directory='/run/borgmatic', local_path=object, remote_path=object, - list_files=True, stream_processes=True, ).and_return((('borg', 'create'), ('repo::archive',), flexmock())) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( diff --git a/tests/unit/actions/test_compact.py b/tests/unit/actions/test_compact.py index 1531fd59..04a3a25e 100644 --- a/tests/unit/actions/test_compact.py +++ b/tests/unit/actions/test_compact.py @@ -9,7 +9,10 @@ def test_compact_actions_calls_hooks_for_configured_repository(): flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never() flexmock(module.borgmatic.borg.compact).should_receive('compact_segments').once() compact_arguments = flexmock( - repository=None, progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock() + repository=None, + progress=flexmock(), + cleanup_commits=flexmock(), + compact_threshold=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -34,7 +37,10 @@ def test_compact_runs_with_selected_repository(): flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.compact).should_receive('compact_segments').once() compact_arguments = flexmock( - repository=flexmock(), progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock() + repository=flexmock(), + progress=flexmock(), + cleanup_commits=flexmock(), + compact_threshold=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -51,84 +57,6 @@ def test_compact_runs_with_selected_repository(): ) -def test_compact_favors_flags_over_config(): - flexmock(module.logger).answer = lambda message: None - flexmock(module.borgmatic.config.validate).should_receive( - 'repositories_match' - ).once().and_return(True) - flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) - flexmock(module.borgmatic.borg.compact).should_receive('compact_segments').with_args( - object, - object, - object, - object, - object, - local_path=object, - remote_path=object, - progress=False, - cleanup_commits=object, - threshold=15, - ).once() - compact_arguments = flexmock( - repository=flexmock(), - progress=False, - cleanup_commits=flexmock(), - threshold=15, - ) - global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) - - module.run_compact( - config_filename='test.yaml', - repository={'path': 'repo'}, - config={'progress': True, 'compact_threshold': 20}, - local_borg_version=None, - compact_arguments=compact_arguments, - global_arguments=global_arguments, - dry_run_label='', - local_path=None, - remote_path=None, - ) - - -def test_compact_favors_defaults_to_config(): - flexmock(module.logger).answer = lambda message: None - flexmock(module.borgmatic.config.validate).should_receive( - 'repositories_match' - ).once().and_return(True) - flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) - flexmock(module.borgmatic.borg.compact).should_receive('compact_segments').with_args( - object, - object, - object, - object, - object, - local_path=object, - remote_path=object, - progress=True, - cleanup_commits=object, - threshold=20, - ).once() - compact_arguments = flexmock( - repository=flexmock(), - progress=None, - cleanup_commits=flexmock(), - threshold=None, - ) - global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) - - module.run_compact( - config_filename='test.yaml', - repository={'path': 'repo'}, - config={'progress': True, 'compact_threshold': 20}, - local_borg_version=None, - compact_arguments=compact_arguments, - global_arguments=global_arguments, - dry_run_label='', - local_path=None, - remote_path=None, - ) - - def test_compact_bails_if_repository_does_not_match(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) @@ -137,7 +65,10 @@ def test_compact_bails_if_repository_does_not_match(): ).once().and_return(False) flexmock(module.borgmatic.borg.compact).should_receive('compact_segments').never() compact_arguments = flexmock( - repository=flexmock(), progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock() + repository=flexmock(), + progress=flexmock(), + cleanup_commits=flexmock(), + compact_threshold=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 6f40fb50..6761f8f4 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -443,9 +443,9 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository(): create_arguments = flexmock( repository=None, progress=flexmock(), - stats=flexmock(), + statistics=flexmock(), json=False, - list_files=flexmock(), + list_details=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -484,9 +484,9 @@ def test_run_create_runs_with_selected_repository(): create_arguments = flexmock( repository=flexmock(), progress=flexmock(), - stats=flexmock(), + statistics=flexmock(), json=False, - list_files=flexmock(), + list_details=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -506,118 +506,6 @@ def test_run_create_runs_with_selected_repository(): ) -def test_run_create_favors_flags_over_config(): - flexmock(module.logger).answer = lambda message: None - flexmock(module.borgmatic.config.validate).should_receive( - 'repositories_match' - ).once().and_return(True) - flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return( - flexmock() - ) - flexmock(module.borgmatic.borg.create).should_receive('create_archive').with_args( - object, - object, - object, - object, - object, - object, - object, - local_path=object, - remote_path=object, - progress=False, - stats=False, - json=object, - list_files=False, - stream_processes=object, - ).once() - flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({}) - flexmock(module.borgmatic.hooks.dispatch).should_receive( - 'call_hooks_even_if_unconfigured' - ).and_return({}) - flexmock(module).should_receive('collect_patterns').and_return(()) - flexmock(module).should_receive('process_patterns').and_return([]) - flexmock(module.os.path).should_receive('join').and_return('/run/borgmatic/bootstrap') - create_arguments = flexmock( - repository=flexmock(), - progress=False, - stats=False, - json=False, - list_files=False, - ) - global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) - - list( - module.run_create( - config_filename='test.yaml', - repository={'path': 'repo'}, - config={'progress': True, 'statistics': True, 'list_details': True}, - config_paths=['/tmp/test.yaml'], - local_borg_version=None, - create_arguments=create_arguments, - global_arguments=global_arguments, - dry_run_label='', - local_path=None, - remote_path=None, - ) - ) - - -def test_run_create_defaults_to_config(): - flexmock(module.logger).answer = lambda message: None - flexmock(module.borgmatic.config.validate).should_receive( - 'repositories_match' - ).once().and_return(True) - flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').and_return( - flexmock() - ) - flexmock(module.borgmatic.borg.create).should_receive('create_archive').with_args( - object, - object, - object, - object, - object, - object, - object, - local_path=object, - remote_path=object, - progress=True, - stats=True, - json=object, - list_files=True, - stream_processes=object, - ).once() - flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({}) - flexmock(module.borgmatic.hooks.dispatch).should_receive( - 'call_hooks_even_if_unconfigured' - ).and_return({}) - flexmock(module).should_receive('collect_patterns').and_return(()) - flexmock(module).should_receive('process_patterns').and_return([]) - flexmock(module.os.path).should_receive('join').and_return('/run/borgmatic/bootstrap') - create_arguments = flexmock( - repository=flexmock(), - progress=True, - stats=True, - json=False, - list_files=True, - ) - global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) - - list( - module.run_create( - config_filename='test.yaml', - repository={'path': 'repo'}, - config={'progress': True, 'statistics': True, 'list_details': True}, - config_paths=['/tmp/test.yaml'], - local_borg_version=None, - create_arguments=create_arguments, - global_arguments=global_arguments, - dry_run_label='', - local_path=None, - remote_path=None, - ) - ) - - def test_run_create_bails_if_repository_does_not_match(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive( @@ -628,9 +516,9 @@ def test_run_create_bails_if_repository_does_not_match(): create_arguments = flexmock( repository=flexmock(), progress=flexmock(), - stats=flexmock(), + statistics=flexmock(), json=False, - list_files=flexmock(), + list_details=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -650,6 +538,72 @@ def test_run_create_bails_if_repository_does_not_match(): ) +def test_run_create_with_both_list_and_json_errors(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive( + 'repositories_match' + ).once().and_return(True) + flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').never() + flexmock(module.borgmatic.borg.create).should_receive('create_archive').never() + create_arguments = flexmock( + repository=flexmock(), + progress=flexmock(), + statistics=flexmock(), + json=True, + list_details=flexmock(), + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + with pytest.raises(ValueError): + list( + module.run_create( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={'list_details': True}, + config_paths=['/tmp/test.yaml'], + local_borg_version=None, + create_arguments=create_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + ) + + +def test_run_create_with_both_list_and_progress_errors(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive( + 'repositories_match' + ).once().and_return(True) + flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').never() + flexmock(module.borgmatic.borg.create).should_receive('create_archive').never() + create_arguments = flexmock( + repository=flexmock(), + progress=flexmock(), + statistics=flexmock(), + json=False, + list_details=flexmock(), + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + with pytest.raises(ValueError): + list( + module.run_create( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={'list_details': True, 'progress': True}, + config_paths=['/tmp/test.yaml'], + local_borg_version=None, + create_arguments=create_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + ) + + def test_run_create_produces_json(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive( @@ -673,9 +627,9 @@ def test_run_create_produces_json(): create_arguments = flexmock( repository=flexmock(), progress=flexmock(), - stats=flexmock(), + statistics=flexmock(), json=True, - list_files=flexmock(), + list_details=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) diff --git a/tests/unit/actions/test_export_tar.py b/tests/unit/actions/test_export_tar.py index b2e91c38..d32aa6fe 100644 --- a/tests/unit/actions/test_export_tar.py +++ b/tests/unit/actions/test_export_tar.py @@ -13,7 +13,7 @@ def test_run_export_tar_does_not_raise(): paths=flexmock(), destination=flexmock(), tar_filter=flexmock(), - list_files=flexmock(), + list_details=flexmock(), strip_components=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -44,7 +44,6 @@ def test_run_export_tar_favors_flags_over_config(): local_path=object, remote_path=object, tar_filter=object, - list_files=False, strip_components=object, ).once() export_tar_arguments = flexmock( @@ -53,7 +52,7 @@ def test_run_export_tar_favors_flags_over_config(): paths=flexmock(), destination=flexmock(), tar_filter=flexmock(), - list_files=False, + list_details=False, strip_components=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -84,7 +83,6 @@ def test_run_export_tar_defaults_to_config(): local_path=object, remote_path=object, tar_filter=object, - list_files=True, strip_components=object, ).once() export_tar_arguments = flexmock( @@ -93,7 +91,7 @@ def test_run_export_tar_defaults_to_config(): paths=flexmock(), destination=flexmock(), tar_filter=flexmock(), - list_files=None, + list_details=None, strip_components=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) diff --git a/tests/unit/actions/test_extract.py b/tests/unit/actions/test_extract.py index 3707c887..1504d4ce 100644 --- a/tests/unit/actions/test_extract.py +++ b/tests/unit/actions/test_extract.py @@ -44,7 +44,6 @@ def test_run_extract_favors_flags_over_config(): remote_path=object, destination_path=object, strip_components=object, - progress=False, ).once() extract_arguments = flexmock( paths=flexmock(), @@ -83,7 +82,6 @@ def test_run_extract_defaults_to_config(): remote_path=object, destination_path=object, strip_components=object, - progress=True, ).once() extract_arguments = flexmock( paths=flexmock(), diff --git a/tests/unit/actions/test_prune.py b/tests/unit/actions/test_prune.py index 762a83c2..b37e50fb 100644 --- a/tests/unit/actions/test_prune.py +++ b/tests/unit/actions/test_prune.py @@ -7,7 +7,7 @@ def test_run_prune_calls_hooks_for_configured_repository(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never() flexmock(module.borgmatic.borg.prune).should_receive('prune_archives').once() - prune_arguments = flexmock(repository=None, stats=flexmock(), list_archives=flexmock()) + prune_arguments = flexmock(repository=None, statistics=flexmock(), list_details=flexmock()) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_prune( @@ -29,7 +29,9 @@ def test_run_prune_runs_with_selected_repository(): 'repositories_match' ).once().and_return(True) flexmock(module.borgmatic.borg.prune).should_receive('prune_archives').once() - prune_arguments = flexmock(repository=flexmock(), stats=flexmock(), list_archives=flexmock()) + prune_arguments = flexmock( + repository=flexmock(), statistics=flexmock(), list_details=flexmock() + ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_prune( @@ -51,7 +53,9 @@ def test_run_prune_bails_if_repository_does_not_match(): 'repositories_match' ).once().and_return(False) flexmock(module.borgmatic.borg.prune).should_receive('prune_archives').never() - prune_arguments = flexmock(repository=flexmock(), stats=flexmock(), list_archives=flexmock()) + prune_arguments = flexmock( + repository=flexmock(), statistics=flexmock(), list_details=flexmock() + ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_prune( diff --git a/tests/unit/actions/test_transfer.py b/tests/unit/actions/test_transfer.py index 03d259be..be4eda25 100644 --- a/tests/unit/actions/test_transfer.py +++ b/tests/unit/actions/test_transfer.py @@ -1,3 +1,4 @@ +import pytest from flexmock import flexmock from borgmatic.actions import transfer as module @@ -6,7 +7,7 @@ from borgmatic.actions import transfer as module def test_run_transfer_does_not_raise(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.borg.transfer).should_receive('transfer_archives') - transfer_arguments = flexmock() + transfer_arguments = flexmock(archive=None) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_transfer( @@ -18,3 +19,21 @@ def test_run_transfer_does_not_raise(): local_path=None, remote_path=None, ) + + +def test_run_transfer_with_archive_and_match_archives_raises(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.borg.transfer).should_receive('transfer_archives') + transfer_arguments = flexmock(archive='foo') + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + with pytest.raises(ValueError): + module.run_transfer( + repository={'path': 'repo'}, + config={'match_archives': 'foo*'}, + local_borg_version=None, + transfer_arguments=transfer_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index 80ce0d94..0a1f5361 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -155,22 +155,6 @@ def test_make_archive_filter_flags_with_data_check_and_prefix_includes_match_arc assert flags == ('--match-archives', 'sh:foo-*') -def test_make_archive_filter_flags_prefers_check_arguments_match_archives_to_config_match_archives(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - 'baz-*', None, '1.2.3' - ).and_return(('--match-archives', 'sh:baz-*')) - - flags = module.make_archive_filter_flags( - '1.2.3', - {'match_archives': 'bar-{now}', 'prefix': ''}, # noqa: FS003 - ('archives',), - check_arguments=flexmock(match_archives='baz-*'), - ) - - assert flags == ('--match-archives', 'sh:baz-*') - - def test_make_archive_filter_flags_with_archives_check_and_empty_prefix_uses_archive_name_format_instead(): flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( @@ -331,42 +315,7 @@ def test_get_repository_id_with_missing_json_keys_raises(): ) -def test_check_archives_favors_progress_flag_over_config(): - config = {'progress': False} - flexmock(module).should_receive('make_check_name_flags').with_args( - {'repository'}, () - ).and_return(()) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) - flexmock(module.environment).should_receive('make_environment') - flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) - flexmock(module).should_receive('execute_command').with_args( - ('borg', 'check', '--progress', 'repo'), - output_file=module.DO_NOT_CAPTURE, - environment=None, - working_directory=None, - borg_local_path='borg', - borg_exit_codes=None, - ).once() - - module.check_archives( - repository_path='repo', - config=config, - local_borg_version='1.2.3', - check_arguments=flexmock( - progress=True, - repair=None, - only_checks=None, - force=None, - match_archives=None, - max_duration=None, - ), - global_arguments=flexmock(log_json=False), - checks={'repository'}, - archive_filter_flags=(), - ) - - -def test_check_archives_defaults_to_progress_config(): +def test_check_archives_with_progress_passes_through_to_borg(): config = {'progress': True} flexmock(module).should_receive('make_check_name_flags').with_args( {'repository'}, () diff --git a/tests/unit/borg/test_compact.py b/tests/unit/borg/test_compact.py index 882dac69..7fff7728 100644 --- a/tests/unit/borg/test_compact.py +++ b/tests/unit/borg/test_compact.py @@ -27,7 +27,7 @@ def insert_execute_command_mock( COMPACT_COMMAND = ('borg', 'compact') -def test_compact_segments_calls_borg_with_parameters(): +def test_compact_segments_calls_borg_with_flags(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('repo',), logging.INFO) @@ -40,7 +40,7 @@ def test_compact_segments_calls_borg_with_parameters(): ) -def test_compact_segments_with_log_info_calls_borg_with_info_parameter(): +def test_compact_segments_with_log_info_calls_borg_with_info_flag(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--info', 'repo'), logging.INFO) insert_logging_mock(logging.INFO) @@ -54,7 +54,7 @@ def test_compact_segments_with_log_info_calls_borg_with_info_parameter(): ) -def test_compact_segments_with_log_debug_calls_borg_with_debug_parameter(): +def test_compact_segments_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--debug', '--show-rc', 'repo'), logging.INFO) insert_logging_mock(logging.DEBUG) @@ -110,7 +110,7 @@ def test_compact_segments_with_exit_codes_calls_borg_using_them(): ) -def test_compact_segments_with_remote_path_calls_borg_with_remote_path_parameters(): +def test_compact_segments_with_remote_path_calls_borg_with_remote_path_flags(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--remote-path', 'borg1', 'repo'), logging.INFO) @@ -124,21 +124,20 @@ def test_compact_segments_with_remote_path_calls_borg_with_remote_path_parameter ) -def test_compact_segments_with_progress_calls_borg_with_progress_parameter(): +def test_compact_segments_with_progress_calls_borg_with_progress_flag(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--progress', 'repo'), logging.INFO) module.compact_segments( dry_run=False, repository_path='repo', - config={}, + config={'progress': True}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - progress=True, ) -def test_compact_segments_with_cleanup_commits_calls_borg_with_cleanup_commits_parameter(): +def test_compact_segments_with_cleanup_commits_calls_borg_with_cleanup_commits_flag(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--cleanup-commits', 'repo'), logging.INFO) @@ -152,21 +151,20 @@ def test_compact_segments_with_cleanup_commits_calls_borg_with_cleanup_commits_p ) -def test_compact_segments_with_threshold_calls_borg_with_threshold_parameter(): +def test_compact_segments_with_threshold_calls_borg_with_threshold_flag(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--threshold', '20', 'repo'), logging.INFO) module.compact_segments( dry_run=False, repository_path='repo', - config={}, + config={'compact_threshold': 20}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - threshold=20, ) -def test_compact_segments_with_umask_calls_borg_with_umask_parameters(): +def test_compact_segments_with_umask_calls_borg_with_umask_flags(): config = {'umask': '077'} flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--umask', '077', 'repo'), logging.INFO) @@ -180,7 +178,7 @@ def test_compact_segments_with_umask_calls_borg_with_umask_parameters(): ) -def test_compact_segments_with_log_json_calls_borg_with_log_json_parameters(): +def test_compact_segments_with_log_json_calls_borg_with_log_json_flags(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--log-json', 'repo'), logging.INFO) @@ -193,7 +191,7 @@ def test_compact_segments_with_log_json_calls_borg_with_log_json_parameters(): ) -def test_compact_segments_with_lock_wait_calls_borg_with_lock_wait_parameters(): +def test_compact_segments_with_lock_wait_calls_borg_with_lock_wait_flags(): config = {'lock_wait': 5} flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--lock-wait', '5', 'repo'), logging.INFO) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index ad7c044e..80b80ee1 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -631,12 +631,12 @@ def test_make_base_create_command_includes_list_flags_in_borg_command(): config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], + 'list_details': True, }, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/run/borgmatic', - list_files=True, ) assert create_flags == ('borg', 'create', '--list', '--filter', 'FOO') @@ -962,7 +962,7 @@ def test_make_base_create_command_with_non_existent_directory_and_source_directo ) -def test_create_archive_calls_borg_with_parameters(): +def test_create_archive_calls_borg_with_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_base_create_command').and_return( @@ -1029,7 +1029,7 @@ def test_create_archive_calls_borg_with_environment(): ) -def test_create_archive_with_log_info_calls_borg_with_info_parameter(): +def test_create_archive_with_log_info_calls_borg_with_info_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_base_create_command').and_return( @@ -1096,7 +1096,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output(): ) -def test_create_archive_with_log_debug_calls_borg_with_debug_parameter(): +def test_create_archive_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_base_create_command').and_return( @@ -1196,7 +1196,6 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats(): local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', - stats=True, ) @@ -1271,7 +1270,7 @@ def test_create_archive_with_exit_codes_calls_borg_using_them(): ) -def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_output_log_level(): +def test_create_archive_with_stats_calls_borg_with_stats_flag_and_answer_output_log_level(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_base_create_command').and_return( @@ -1296,12 +1295,12 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_ou 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'statistics': True, }, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', - stats=True, ) @@ -1334,16 +1333,16 @@ def test_create_archive_with_files_calls_borg_with_answer_output_log_level(): 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'list_details': True, }, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', - list_files=True, ) -def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_parameter_and_no_list(): +def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_flag_and_no_list(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_base_create_command').and_return( @@ -1369,16 +1368,16 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'progress': True, }, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', - progress=True, ) -def test_create_archive_with_progress_calls_borg_with_progress_parameter(): +def test_create_archive_with_progress_calls_borg_with_progress_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_base_create_command').and_return( @@ -1403,16 +1402,16 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter(): 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'progress': True, }, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', - progress=True, ) -def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progress_parameter(): +def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progress_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER processes = flexmock() @@ -1459,12 +1458,12 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'progress': True, }, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', - progress=True, stream_processes=processes, ) @@ -1532,7 +1531,6 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_flag(): global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', json=True, - stats=True, ) assert json_output == '[]' diff --git a/tests/unit/borg/test_delete.py b/tests/unit/borg/test_delete.py index f6ace21d..770f411d 100644 --- a/tests/unit/borg/test_delete.py +++ b/tests/unit/borg/test_delete.py @@ -21,7 +21,7 @@ def test_make_delete_command_includes_log_info(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -43,7 +43,7 @@ def test_make_delete_command_includes_log_debug(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -67,7 +67,7 @@ def test_make_delete_command_includes_dry_run(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=True, log_json=False), local_path='borg', remote_path=None, @@ -91,7 +91,7 @@ def test_make_delete_command_includes_remote_path(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path='borg1', @@ -114,7 +114,7 @@ def test_make_delete_command_includes_umask(): repository={'path': 'repo'}, config={'umask': '077'}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -138,7 +138,7 @@ def test_make_delete_command_includes_log_json(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=True), local_path='borg', remote_path=None, @@ -162,7 +162,7 @@ def test_make_delete_command_includes_lock_wait(): repository={'path': 'repo'}, config={'lock_wait': 5}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -171,31 +171,7 @@ def test_make_delete_command_includes_lock_wait(): assert command == ('borg', 'delete', '--lock-wait', '5', 'repo') -def test_make_delete_command_favors_list_flag_over_config(): - flexmock(module.borgmatic.borg.flags).should_receive('make_flags').and_return(()) - flexmock(module.borgmatic.borg.flags).should_receive('make_flags').with_args( - 'list', True - ).and_return(('--list',)) - flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) - flexmock(module.borgmatic.borg.flags).should_receive('make_flags_from_arguments').and_return(()) - flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( - ('repo',) - ) - - command = module.make_delete_command( - repository={'path': 'repo'}, - config={'list_details': False}, - local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=True, force=0, match_archives=None, archive=None), - global_arguments=flexmock(dry_run=False, log_json=False), - local_path='borg', - remote_path=None, - ) - - assert command == ('borg', 'delete', '--list', 'repo') - - -def test_make_delete_command_defaults_to_list_config(): +def test_make_delete_command_with_list_config_calls_borg_with_list_flag(): flexmock(module.borgmatic.borg.flags).should_receive('make_flags').and_return(()) flexmock(module.borgmatic.borg.flags).should_receive('make_flags').with_args( 'list', True @@ -210,7 +186,7 @@ def test_make_delete_command_defaults_to_list_config(): repository={'path': 'repo'}, config={'list_details': True}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=None, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=None, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -231,7 +207,7 @@ def test_make_delete_command_includes_force(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=1, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=1, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -252,7 +228,7 @@ def test_make_delete_command_includes_force_twice(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=2, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=2, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -276,7 +252,7 @@ def test_make_delete_command_includes_archive(): config={}, local_borg_version='1.2.3', delete_arguments=flexmock( - list_archives=False, force=0, match_archives=None, archive='archive' + list_details=False, force=0, match_archives=None, archive='archive' ), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', @@ -301,7 +277,7 @@ def test_make_delete_command_includes_match_archives(): config={}, local_borg_version='1.2.3', delete_arguments=flexmock( - list_archives=False, force=0, match_archives='sh:foo*', archive='archive' + list_details=False, force=0, match_archives='sh:foo*', archive='archive' ), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', @@ -390,7 +366,7 @@ def test_delete_archives_without_archive_related_argument_calls_borg_repo_delete config={}, local_borg_version=flexmock(), delete_arguments=flexmock( - list_archives=True, force=False, cache_only=False, keep_security_info=False + list_details=True, force=False, cache_only=False, keep_security_info=False ), global_arguments=flexmock(), ) diff --git a/tests/unit/borg/test_export_tar.py b/tests/unit/borg/test_export_tar.py index f45026f3..3fb50129 100644 --- a/tests/unit/borg/test_export_tar.py +++ b/tests/unit/borg/test_export_tar.py @@ -144,7 +144,7 @@ def test_export_tar_archive_calls_borg_with_umask_flags(): ) -def test_export_tar_archive_calls_borg_with_log_json_parameter(): +def test_export_tar_archive_calls_borg_with_log_json_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( @@ -186,7 +186,7 @@ def test_export_tar_archive_calls_borg_with_lock_wait_flags(): ) -def test_export_tar_archive_with_log_info_calls_borg_with_info_parameter(): +def test_export_tar_archive_with_log_info_calls_borg_with_info_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( @@ -230,7 +230,7 @@ def test_export_tar_archive_with_log_debug_calls_borg_with_debug_flags(): ) -def test_export_tar_archive_calls_borg_with_dry_run_parameter(): +def test_export_tar_archive_calls_borg_with_dry_run_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( @@ -273,7 +273,7 @@ def test_export_tar_archive_calls_borg_with_tar_filter_flags(): ) -def test_export_tar_archive_calls_borg_with_list_parameter(): +def test_export_tar_archive_calls_borg_with_list_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( @@ -290,14 +290,13 @@ def test_export_tar_archive_calls_borg_with_list_parameter(): archive='archive', paths=None, destination_path='test.tar', - config={}, + config={'list_details': True}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - list_files=True, ) -def test_export_tar_archive_calls_borg_with_strip_components_parameter(): +def test_export_tar_archive_calls_borg_with_strip_components_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( @@ -320,7 +319,7 @@ def test_export_tar_archive_calls_borg_with_strip_components_parameter(): ) -def test_export_tar_archive_skips_abspath_for_remote_repository_parameter(): +def test_export_tar_archive_skips_abspath_for_remote_repository_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( diff --git a/tests/unit/borg/test_extract.py b/tests/unit/borg/test_extract.py index de392f24..a11155f8 100644 --- a/tests/unit/borg/test_extract.py +++ b/tests/unit/borg/test_extract.py @@ -580,7 +580,7 @@ def test_extract_archive_with_strip_components_all_and_no_paths_raises(): ) -def test_extract_archive_calls_borg_with_progress_parameter(): +def test_extract_archive_calls_borg_with_progress_flag(): flexmock(module.os.path).should_receive('abspath').and_return('repo') flexmock(module.environment).should_receive('make_environment') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) @@ -606,10 +606,9 @@ def test_extract_archive_calls_borg_with_progress_parameter(): repository='repo', archive='archive', paths=None, - config={}, + config={'progress': True}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - progress=True, ) @@ -622,10 +621,9 @@ def test_extract_archive_with_progress_and_extract_to_stdout_raises(): repository='repo', archive='archive', paths=None, - config={}, + config={'progress': True}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - progress=True, extract_to_stdout=True, ) diff --git a/tests/unit/borg/test_info.py b/tests/unit/borg/test_info.py index 5866b4ab..23f416bb 100644 --- a/tests/unit/borg/test_info.py +++ b/tests/unit/borg/test_info.py @@ -380,7 +380,7 @@ def test_make_info_command_with_match_archives_flag_passes_through_to_command(): command = module.make_info_command( repository_path='repo', - config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + config={'archive_name_format': 'bar-{now}', 'match_archives': 'sh:foo-*'}, # noqa: FS003 local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives='sh:foo-*'), diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index 1d7888a3..10e8ebf9 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -135,32 +135,6 @@ def test_make_prune_flags_without_prefix_uses_archive_name_format_instead(): assert result == expected -def test_make_prune_flags_without_prefix_uses_match_archives_flag_instead_of_option(): - config = { - 'archive_name_format': 'bar-{now}', # noqa: FS003 - 'match_archives': 'foo*', - 'keep_daily': 1, - 'prefix': None, - } - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - 'baz*', 'bar-{now}', '1.2.3' # noqa: FS003 - ).and_return(('--match-archives', 'sh:bar-*')).once() - - result = module.make_prune_flags( - config, flexmock(match_archives='baz*'), local_borg_version='1.2.3' - ) - - expected = ( - '--keep-daily', - '1', - '--match-archives', - 'sh:bar-*', # noqa: FS003 - ) - - assert result == expected - - def test_make_prune_flags_without_prefix_uses_match_archives_option(): config = { 'archive_name_format': 'bar-{now}', # noqa: FS003 @@ -215,7 +189,7 @@ def test_prune_archives_calls_borg_with_flags(): ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('repo',), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -237,7 +211,7 @@ def test_prune_archives_with_log_info_calls_borg_with_info_flag(): insert_execute_command_mock(PRUNE_COMMAND + ('--info', 'repo'), logging.INFO) insert_logging_mock(logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( repository_path='repo', config={}, @@ -259,7 +233,7 @@ def test_prune_archives_with_log_debug_calls_borg_with_debug_flag(): insert_execute_command_mock(PRUNE_COMMAND + ('--debug', '--show-rc', 'repo'), logging.INFO) insert_logging_mock(logging.DEBUG) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( repository_path='repo', config={}, @@ -280,7 +254,7 @@ def test_prune_archives_with_dry_run_calls_borg_with_dry_run_flag(): ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--dry-run', 'repo'), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( repository_path='repo', config={}, @@ -301,7 +275,7 @@ def test_prune_archives_with_local_path_calls_borg_via_local_path(): ).and_return(False) insert_execute_command_mock(('borg1',) + PRUNE_COMMAND[1:] + ('repo',), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -328,7 +302,7 @@ def test_prune_archives_with_exit_codes_calls_borg_using_them(): borg_exit_codes=borg_exit_codes, ) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -349,7 +323,7 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_flags(): ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--remote-path', 'borg1', 'repo'), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -361,7 +335,7 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_flags(): ) -def test_prune_archives_favors_stats_flag_over_config(): +def test_prune_archives_with_stats_config_calls_borg_with_stats_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -371,28 +345,7 @@ def test_prune_archives_favors_stats_flag_over_config(): ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--stats', 'repo'), module.borgmatic.logger.ANSWER) - prune_arguments = flexmock(stats=True, list_archives=False) - module.prune_archives( - dry_run=False, - repository_path='repo', - config={'statistics': False}, - local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), - prune_arguments=prune_arguments, - ) - - -def test_prune_archives_defaults_to_stats_config(): - flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') - flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) - flexmock(module.feature).should_receive('available').with_args( - module.feature.Feature.NO_PRUNE_STATS, '1.2.3' - ).and_return(False) - insert_execute_command_mock(PRUNE_COMMAND + ('--stats', 'repo'), module.borgmatic.logger.ANSWER) - - prune_arguments = flexmock(stats=None, list_archives=False) + prune_arguments = flexmock(statistics=None, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -403,7 +356,7 @@ def test_prune_archives_defaults_to_stats_config(): ) -def test_prune_archives_favors_list_flag_over_config(): +def test_prune_archives_with_list_config_calls_borg_with_list_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -413,28 +366,7 @@ def test_prune_archives_favors_list_flag_over_config(): ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--list', 'repo'), module.borgmatic.logger.ANSWER) - prune_arguments = flexmock(stats=False, list_archives=True) - module.prune_archives( - dry_run=False, - repository_path='repo', - config={'list_details': False}, - local_borg_version='1.2.3', - global_arguments=flexmock(log_json=False), - prune_arguments=prune_arguments, - ) - - -def test_prune_archives_defaults_to_list_config(): - flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') - flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) - flexmock(module.feature).should_receive('available').with_args( - module.feature.Feature.NO_PRUNE_STATS, '1.2.3' - ).and_return(False) - insert_execute_command_mock(PRUNE_COMMAND + ('--list', 'repo'), module.borgmatic.logger.ANSWER) - - prune_arguments = flexmock(stats=False, list_archives=None) + prune_arguments = flexmock(statistics=False, list_details=None) module.prune_archives( dry_run=False, repository_path='repo', @@ -456,7 +388,7 @@ def test_prune_archives_with_umask_calls_borg_with_umask_flags(): ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--umask', '077', 'repo'), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -477,7 +409,7 @@ def test_prune_archives_with_log_json_calls_borg_with_log_json_flag(): ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--log-json', 'repo'), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -499,7 +431,7 @@ def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--lock-wait', '5', 'repo'), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -520,7 +452,7 @@ def test_prune_archives_with_extra_borg_options_calls_borg_with_extra_options(): ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--extra', '--options', 'repo'), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -588,7 +520,7 @@ def test_prune_archives_with_date_based_matching_calls_borg_with_date_based_flag ) prune_arguments = flexmock( - stats=False, list_archives=False, newer='1d', newest='1y', older='1m', oldest='1w' + statistics=False, list_details=False, newer='1d', newest='1y', older='1m', oldest='1w' ) module.prune_archives( dry_run=False, @@ -612,7 +544,7 @@ def test_prune_archives_calls_borg_with_working_directory(): PRUNE_COMMAND + ('repo',), logging.INFO, working_directory='/working/dir' ) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -623,7 +555,7 @@ def test_prune_archives_calls_borg_with_working_directory(): ) -def test_prune_archives_calls_borg_with_flags_and_when_feature_available(): +def test_prune_archives_calls_borg_without_stats_when_feature_is_not_available(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) @@ -633,11 +565,11 @@ def test_prune_archives_calls_borg_with_flags_and_when_feature_available(): ).and_return(True) insert_execute_command_mock(PRUNE_COMMAND + ('repo',), logging.ANSWER) - prune_arguments = flexmock(stats=True, list_archives=False) + prune_arguments = flexmock(statistics=True, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', - config={}, + config={'statistics': True}, local_borg_version='2.0.0b10', global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, diff --git a/tests/unit/borg/test_recreate.py b/tests/unit/borg/test_recreate.py index dd91218d..c1f135b2 100644 --- a/tests/unit/borg/test_recreate.py +++ b/tests/unit/borg/test_recreate.py @@ -225,37 +225,7 @@ def test_recreate_with_log_json(): ) -def test_recreate_archive_favors_list_flag_over_config(): - flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) - flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) - flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) - flexmock(module.borgmatic.borg.flags).should_receive( - 'make_repository_archive_flags' - ).and_return(('repo::archive',)) - flexmock(module).should_receive('make_list_filter_flags').and_return('AME+-') - insert_execute_command_mock( - ('borg', 'recreate', '--list', '--filter', 'AME+-', 'repo::archive') - ) - - module.recreate_archive( - repository='repo', - archive='archive', - config={'list_details': False}, - local_borg_version='1.2.3', - recreate_arguments=flexmock( - list=True, - target=None, - comment=None, - timestamp=None, - match_archives=None, - ), - global_arguments=flexmock(dry_run=False, log_json=False), - local_path='borg', - patterns=None, - ) - - -def test_recreate_archive_defaults_to_list_config(): +def test_recreate_with_list_config_calls_borg_with_list_flag(): flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) diff --git a/tests/unit/borg/test_repo_delete.py b/tests/unit/borg/test_repo_delete.py index 66e73778..4af66085 100644 --- a/tests/unit/borg/test_repo_delete.py +++ b/tests/unit/borg/test_repo_delete.py @@ -19,7 +19,7 @@ def test_make_repo_delete_command_with_feature_available_runs_borg_repo_delete() repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -40,7 +40,7 @@ def test_make_repo_delete_command_without_feature_available_runs_borg_delete(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -62,7 +62,7 @@ def test_make_repo_delete_command_includes_log_info(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -84,7 +84,7 @@ def test_make_repo_delete_command_includes_log_debug(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -108,7 +108,7 @@ def test_make_repo_delete_command_includes_dry_run(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=True, log_json=False), local_path='borg', remote_path=None, @@ -132,7 +132,7 @@ def test_make_repo_delete_command_includes_remote_path(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path='borg1', @@ -155,7 +155,7 @@ def test_make_repo_delete_command_includes_umask(): repository={'path': 'repo'}, config={'umask': '077'}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -179,7 +179,7 @@ def test_make_repo_delete_command_includes_log_json(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=True), local_path='borg', remote_path=None, @@ -203,7 +203,7 @@ def test_make_repo_delete_command_includes_lock_wait(): repository={'path': 'repo'}, config={'lock_wait': 5}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -225,9 +225,9 @@ def test_make_repo_delete_command_includes_list(): command = module.make_repo_delete_command( repository={'path': 'repo'}, - config={}, + config={'list_details': True}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=True, force=0), + repo_delete_arguments=flexmock(list_details=True, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -248,7 +248,7 @@ def test_make_repo_delete_command_includes_force(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=1), + repo_delete_arguments=flexmock(list_details=False, force=1), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -269,7 +269,7 @@ def test_make_repo_delete_command_includes_force_twice(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=2), + repo_delete_arguments=flexmock(list_details=False, force=2), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, diff --git a/tests/unit/borg/test_repo_list.py b/tests/unit/borg/test_repo_list.py index da750d7a..73f567f5 100644 --- a/tests/unit/borg/test_repo_list.py +++ b/tests/unit/borg/test_repo_list.py @@ -664,7 +664,7 @@ def test_make_repo_list_command_with_match_archives_calls_borg_with_match_archiv command = module.make_repo_list_command( repository_path='repo', - config={}, + config={'match_archives': 'foo-*'}, local_borg_version='1.2.3', repo_list_arguments=flexmock( archive=None, diff --git a/tests/unit/borg/test_transfer.py b/tests/unit/borg/test_transfer.py index 9e6618e5..ced41f3c 100644 --- a/tests/unit/borg/test_transfer.py +++ b/tests/unit/borg/test_transfer.py @@ -193,7 +193,7 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl module.transfer_archives( dry_run=False, repository_path='repo', - config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + config={'archive_name_format': 'bar-{now}', 'match_archives': 'sh:foo*'}, # noqa: FS003 local_borg_version='2.3.4', transfer_arguments=flexmock( archive=None, progress=None, match_archives='sh:foo*', source_repository=None @@ -436,38 +436,7 @@ def test_transfer_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): ) -def test_transfer_archives_favors_progress_flag_over_config(): - flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') - flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER - flexmock(module.flags).should_receive('make_flags').and_return(()) - flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--progress',)) - flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) - flexmock(module.environment).should_receive('make_environment') - flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) - flexmock(module).should_receive('execute_command').with_args( - ('borg', 'transfer', '--progress', '--repo', 'repo'), - output_log_level=module.borgmatic.logger.ANSWER, - output_file=module.DO_NOT_CAPTURE, - environment=None, - working_directory=None, - borg_local_path='borg', - borg_exit_codes=None, - ) - - module.transfer_archives( - dry_run=False, - repository_path='repo', - config={'progress': False}, - local_borg_version='2.3.4', - transfer_arguments=flexmock( - archive=None, progress=True, match_archives=None, source_repository=None - ), - global_arguments=flexmock(log_json=False), - ) - - -def test_transfer_archives_defaults_to_progress_flag(): +def test_transfer_archives_with_progress_calls_borg_with_progress_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 18793f19..57f3cf67 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -1578,7 +1578,7 @@ def test_load_configurations_collects_parsed_configurations_and_logs(resolve_env configs, config_paths, logs = tuple( module.load_configurations( ('test.yaml', 'other.yaml'), - global_arguments=flexmock(), + arguments=flexmock(), resolve_env=resolve_env, ) ) @@ -1592,7 +1592,7 @@ def test_load_configurations_logs_warning_for_permission_error(): flexmock(module.validate).should_receive('parse_configuration').and_raise(PermissionError) configs, config_paths, logs = tuple( - module.load_configurations(('test.yaml',), global_arguments=flexmock()) + module.load_configurations(('test.yaml',), arguments=flexmock()) ) assert configs == {} @@ -1604,7 +1604,7 @@ def test_load_configurations_logs_critical_for_parse_error(): flexmock(module.validate).should_receive('parse_configuration').and_raise(ValueError) configs, config_paths, logs = tuple( - module.load_configurations(('test.yaml',), global_arguments=flexmock()) + module.load_configurations(('test.yaml',), arguments=flexmock()) ) assert configs == {} diff --git a/tests/unit/config/test_arguments.py b/tests/unit/config/test_arguments.py index 18ad7f56..2c07f815 100644 --- a/tests/unit/config/test_arguments.py +++ b/tests/unit/config/test_arguments.py @@ -189,3 +189,15 @@ def test_prepare_arguments_for_config_skips_option_missing_from_schema(): }, }, ) == ((('other_option',), 'value2'),) + + +def test_apply_arguments_to_config_does_not_raise(): + flexmock(module).should_receive('prepare_arguments_for_config').and_return( + ( + (('foo', 'bar'), 'baz'), + (('one', 'two'), 'three'), + ) + ) + flexmock(module).should_receive('set_values') + + module.apply_arguments_to_config(config={}, schema={}, arguments={'global': flexmock()}) From 10fb02c40ad19762563a8ccd79a84f95efa51c6a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 31 Mar 2025 13:33:39 -0700 Subject: [PATCH 205/226] Fix bootstrap --progress flag (#303). --- borgmatic/actions/config/bootstrap.py | 4 +++- tests/unit/actions/config/test_bootstrap.py | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/borgmatic/actions/config/bootstrap.py b/borgmatic/actions/config/bootstrap.py index 74714c7e..520bcb1c 100644 --- a/borgmatic/actions/config/bootstrap.py +++ b/borgmatic/actions/config/bootstrap.py @@ -119,7 +119,9 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version): bootstrap_arguments.repository, archive_name, [config_path.lstrip(os.path.sep) for config_path in manifest_config_paths], - config, + # Only add progress here and not the extract_archive() call above, because progress + # conflicts with extract_to_stdout. + dict(config, progress=bootstrap_arguments.progress or False), local_borg_version, global_arguments, local_path=bootstrap_arguments.local_path, diff --git a/tests/unit/actions/config/test_bootstrap.py b/tests/unit/actions/config/test_bootstrap.py index 554ad05d..ece91197 100644 --- a/tests/unit/actions/config/test_bootstrap.py +++ b/tests/unit/actions/config/test_bootstrap.py @@ -105,7 +105,7 @@ def test_get_config_paths_translates_ssh_command_argument_to_config(): flexmock(module.borgmatic.config.paths).should_receive( 'get_borgmatic_source_directory' ).and_return('/source') - config = flexmock() + config = {} flexmock(module).should_receive('make_bootstrap_config').and_return(config) bootstrap_arguments = flexmock( repository='repo', @@ -271,6 +271,7 @@ def test_run_bootstrap_does_not_raise(): ssh_command=None, local_path='borg7', remote_path='borg8', + progress=None, ) global_arguments = flexmock( dry_run=False, @@ -298,7 +299,7 @@ def test_run_bootstrap_does_not_raise(): def test_run_bootstrap_translates_ssh_command_argument_to_config(): - config = flexmock() + config = {} flexmock(module).should_receive('make_bootstrap_config').and_return(config) flexmock(module).should_receive('get_config_paths').and_return(['/borgmatic/config.yaml']) bootstrap_arguments = flexmock( @@ -310,6 +311,7 @@ def test_run_bootstrap_translates_ssh_command_argument_to_config(): ssh_command='ssh -i key', local_path='borg7', remote_path='borg8', + progress=None, ) global_arguments = flexmock( dry_run=False, @@ -331,7 +333,7 @@ def test_run_bootstrap_translates_ssh_command_argument_to_config(): 'repo', 'archive', object, - config, + {'progress': False}, object, object, extract_to_stdout=False, From f166111b9bde9820e40fa1b919d4be6c1022581a Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 31 Mar 2025 15:26:24 -0700 Subject: [PATCH 206/226] Fix new "repositories:" sub-options ("append_only", "make_parent_directories", etc.) (#303). --- borgmatic/actions/repo_create.py | 6 +++--- borgmatic/borg/repo_create.py | 4 ++-- borgmatic/commands/arguments.py | 3 +++ borgmatic/config/schema.yaml | 18 +++++++++++++++- tests/unit/actions/test_repo_create.py | 20 ++++++++--------- tests/unit/borg/test_repo_create.py | 30 ++++++++++++++++++++++---- 6 files changed, 61 insertions(+), 20 deletions(-) diff --git a/borgmatic/actions/repo_create.py b/borgmatic/actions/repo_create.py index 5166b9b8..38d35922 100644 --- a/borgmatic/actions/repo_create.py +++ b/borgmatic/actions/repo_create.py @@ -52,9 +52,9 @@ def run_repo_create( else repo_create_arguments.storage_quota ), ( - repository.get('make_parent_dirs') - if repo_create_arguments.make_parent_dirs is None - else repo_create_arguments.make_parent_dirs + repository.get('make_parent_directories') + if repo_create_arguments.make_parent_directories is None + else repo_create_arguments.make_parent_directories ), local_path=local_path, remote_path=remote_path, diff --git a/borgmatic/borg/repo_create.py b/borgmatic/borg/repo_create.py index b4cde394..9ed2619e 100644 --- a/borgmatic/borg/repo_create.py +++ b/borgmatic/borg/repo_create.py @@ -24,7 +24,7 @@ def create_repository( copy_crypt_key=False, append_only=None, storage_quota=None, - make_parent_dirs=False, + make_parent_directories=False, local_path='borg', remote_path=None, ): @@ -79,7 +79,7 @@ def create_repository( + (('--copy-crypt-key',) if copy_crypt_key else ()) + (('--append-only',) if append_only else ()) + (('--storage-quota', storage_quota) if storage_quota else ()) - + (('--make-parent-dirs',) if make_parent_dirs else ()) + + (('--make-parent-dirs',) if make_parent_directories else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ()) + (('--log-json',) if global_arguments.log_json else ()) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index c3b61f42..6bb4fc46 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -666,6 +666,7 @@ def make_parsers(schema, unparsed_arguments): ) repo_create_group.add_argument( '--append-only', + default=None, action='store_true', help='Create an append-only repository', ) @@ -675,6 +676,8 @@ def make_parsers(schema, unparsed_arguments): ) repo_create_group.add_argument( '--make-parent-dirs', + dest='make_parent_directories', + default=None, action='store_true', help='Create any missing parent directories of the repository directory', ) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 3242bdbb..d5c572f0 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -33,6 +33,7 @@ properties: type: object required: - path + additionalProperties: false properties: path: type: string @@ -67,7 +68,7 @@ properties: only used for the repo-create action. Defaults to no quota. example: 5G - make_parent_dirs: + make_parent_directories: type: boolean description: | Whether any missing parent directories of the repository @@ -1187,6 +1188,7 @@ properties: run: [echo Backing up.] bootstrap: type: object + additionalProperties: false properties: store_config_files: type: boolean @@ -1918,6 +1920,7 @@ properties: example: tk_AgQdq7mVBoFD37zQVN29RhuMzNIz2 start: type: object + additionalProperties: false properties: title: type: string @@ -1941,6 +1944,7 @@ properties: example: incoming_envelope finish: type: object + additionalProperties: false properties: title: type: string @@ -1964,6 +1968,7 @@ properties: example: incoming_envelope fail: type: object + additionalProperties: false properties: title: type: string @@ -2022,6 +2027,7 @@ properties: example: hwRwoWsXMBWwgrSecfa9EfPey55WSN start: type: object + additionalProperties: false properties: message: type: string @@ -2097,6 +2103,7 @@ properties: example: Pushover Link finish: type: object + additionalProperties: false properties: message: type: string @@ -2172,6 +2179,7 @@ properties: example: Pushover Link fail: type: object + additionalProperties: false properties: message: type: string @@ -2311,6 +2319,7 @@ properties: example: fakekey start: type: object + additionalProperties: false properties: value: type: ["integer", "string"] @@ -2319,6 +2328,7 @@ properties: example: STARTED finish: type: object + additionalProperties: false properties: value: type: ["integer", "string"] @@ -2327,6 +2337,7 @@ properties: example: FINISH fail: type: object + additionalProperties: false properties: value: type: ["integer", "string"] @@ -2358,6 +2369,7 @@ properties: type: array items: type: object + additionalProperties: false required: - url - label @@ -2399,6 +2411,7 @@ properties: start: type: object required: ['body'] + additionalProperties: false properties: title: type: string @@ -2414,6 +2427,7 @@ properties: finish: type: object required: ['body'] + additionalProperties: false properties: title: type: string @@ -2429,6 +2443,7 @@ properties: fail: type: object required: ['body'] + additionalProperties: false properties: title: type: string @@ -2444,6 +2459,7 @@ properties: log: type: object required: ['body'] + additionalProperties: false properties: title: type: string diff --git a/tests/unit/actions/test_repo_create.py b/tests/unit/actions/test_repo_create.py index 665d9db7..0b54818e 100644 --- a/tests/unit/actions/test_repo_create.py +++ b/tests/unit/actions/test_repo_create.py @@ -15,7 +15,7 @@ def test_run_repo_create_with_encryption_mode_argument_does_not_raise(): copy_crypt_key=flexmock(), append_only=flexmock(), storage_quota=flexmock(), - make_parent_dirs=flexmock(), + make_parent_directories=flexmock(), ) module.run_repo_create( @@ -40,7 +40,7 @@ def test_run_repo_create_with_encryption_mode_option_does_not_raise(): copy_crypt_key=flexmock(), append_only=flexmock(), storage_quota=flexmock(), - make_parent_dirs=flexmock(), + make_parent_directories=flexmock(), ) module.run_repo_create( @@ -65,7 +65,7 @@ def test_run_repo_create_without_encryption_mode_raises(): copy_crypt_key=flexmock(), append_only=flexmock(), storage_quota=flexmock(), - make_parent_dirs=flexmock(), + make_parent_directories=flexmock(), ) with pytest.raises(ValueError): @@ -93,7 +93,7 @@ def test_run_repo_create_bails_if_repository_does_not_match(): copy_crypt_key=flexmock(), append_only=flexmock(), storage_quota=flexmock(), - make_parent_dirs=flexmock(), + make_parent_directories=flexmock(), ) module.run_repo_create( @@ -121,7 +121,7 @@ def test_run_repo_create_favors_flags_over_config(): object, append_only=False, storage_quota=0, - make_parent_dirs=False, + make_parent_directories=False, local_path=object, remote_path=object, ).once() @@ -132,7 +132,7 @@ def test_run_repo_create_favors_flags_over_config(): copy_crypt_key=flexmock(), append_only=False, storage_quota=0, - make_parent_dirs=False, + make_parent_directories=False, ) module.run_repo_create( @@ -140,7 +140,7 @@ def test_run_repo_create_favors_flags_over_config(): 'path': 'repo', 'append_only': True, 'storage_quota': '10G', - 'make_parent_dirs': True, + 'make_parent_directories': True, }, config={}, local_borg_version=None, @@ -165,7 +165,7 @@ def test_run_repo_create_defaults_to_config(): object, append_only=True, storage_quota='10G', - make_parent_dirs=True, + make_parent_directories=True, local_path=object, remote_path=object, ).once() @@ -176,7 +176,7 @@ def test_run_repo_create_defaults_to_config(): copy_crypt_key=flexmock(), append_only=None, storage_quota=None, - make_parent_dirs=None, + make_parent_directories=None, ) module.run_repo_create( @@ -184,7 +184,7 @@ def test_run_repo_create_defaults_to_config(): 'path': 'repo', 'append_only': True, 'storage_quota': '10G', - 'make_parent_dirs': True, + 'make_parent_directories': True, }, config={}, local_borg_version=None, diff --git a/tests/unit/borg/test_repo_create.py b/tests/unit/borg/test_repo_create.py index dc645f12..9e3a674d 100644 --- a/tests/unit/borg/test_repo_create.py +++ b/tests/unit/borg/test_repo_create.py @@ -228,7 +228,29 @@ def test_create_repository_with_append_only_calls_borg_with_append_only_flag(): module.create_repository( dry_run=False, repository_path='repo', - config={}, + config={'append_only': True}, + local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), + encryption_mode='repokey', + append_only=True, + ) + + +def test_create_repository_with_append_only_config_calls_borg_with_append_only_flag(): + insert_repo_info_command_not_found_mock() + insert_repo_create_command_mock(REPO_CREATE_COMMAND + ('--append-only', '--repo', 'repo')) + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) + + module.create_repository( + dry_run=False, + repository_path='repo', + config={'append_only': True}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', @@ -252,7 +274,7 @@ def test_create_repository_with_storage_quota_calls_borg_with_storage_quota_flag module.create_repository( dry_run=False, repository_path='repo', - config={}, + config={'storage_quota': '5G'}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', @@ -274,11 +296,11 @@ def test_create_repository_with_make_parent_dirs_calls_borg_with_make_parent_dir module.create_repository( dry_run=False, repository_path='repo', - config={}, + config={'make_parent_directories': True}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', - make_parent_dirs=True, + make_parent_directories=True, ) From 8c907bb5a344741e4dd5993c6d26f485984ec13e Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 31 Mar 2025 17:11:37 -0700 Subject: [PATCH 207/226] Fix broken "recreate" action with Borg 1.4 (#610). --- borgmatic/borg/recreate.py | 29 +-- borgmatic/commands/arguments.py | 2 +- tests/unit/borg/test_recreate.py | 300 ++++++++++++++++++++++++------- 3 files changed, 252 insertions(+), 79 deletions(-) diff --git a/borgmatic/borg/recreate.py b/borgmatic/borg/recreate.py index bea11b58..c0ff0704 100644 --- a/borgmatic/borg/recreate.py +++ b/borgmatic/borg/recreate.py @@ -2,6 +2,7 @@ import logging import shlex import borgmatic.borg.environment +import borgmatic.borg.feature import borgmatic.config.paths import borgmatic.execute from borgmatic.borg import flags @@ -68,21 +69,25 @@ def recreate_archive( + (('--timestamp', recreate_arguments.timestamp) if recreate_arguments.timestamp else ()) + (('--compression', compression) if compression else ()) + (('--chunker-params', chunker_params) if chunker_params else ()) - + ( - flags.make_match_archives_flags( - recreate_arguments.match_archives or archive or config.get('match_archives'), - config.get('archive_name_format'), - local_borg_version, - ) - if recreate_arguments.match_archives - else () - ) + (('--recompress', recompress) if recompress else ()) + exclude_flags + ( - flags.make_repository_archive_flags(repository, archive, local_borg_version) - if archive - else flags.make_repository_flags(repository, local_borg_version) + ( + flags.make_repository_flags(repository, local_borg_version) + + flags.make_match_archives_flags( + archive or config.get('match_archives'), + config.get('archive_name_format'), + local_borg_version, + ) + ) + if borgmatic.borg.feature.available( + borgmatic.borg.feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version + ) + else ( + flags.make_repository_archive_flags(repository, archive, local_borg_version) + if archive + else flags.make_repository_flags(repository, local_borg_version) + ) ) ) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 5b900d47..671893c4 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1562,7 +1562,7 @@ def make_parsers(): '--glob-archives', dest='match_archives', metavar='PATTERN', - help='Only consider archive names, hashes, or series matching this pattern', + help='Only consider archive names, hashes, or series matching this pattern [Borg 2.x+ only]', ) recreate_group.add_argument( '-h', '--help', action='help', help='Show this help message and exit' diff --git a/tests/unit/borg/test_recreate.py b/tests/unit/borg/test_recreate.py index fe2739b7..d65a86ed 100644 --- a/tests/unit/borg/test_recreate.py +++ b/tests/unit/borg/test_recreate.py @@ -20,24 +20,20 @@ def insert_execute_command_mock(command, working_directory=None, borg_exit_codes ).once() -def mock_dependencies(): - flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) - flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) - flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') - flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) - flexmock(module.borgmatic.borg.flags).should_receive( - 'make_repository_archive_flags' - ).and_return(('repo::archive',)) - - def test_recreate_archive_dry_run_skips_execution(): flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.flags).should_receive( 'make_repository_archive_flags' - ).and_return(('repo::archive',)) + ).and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module.borgmatic.execute).should_receive('execute_command').never() recreate_arguments = flexmock( @@ -67,10 +63,16 @@ def test_recreate_calls_borg_with_required_flags(): flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.flags).should_receive( 'make_repository_archive_flags' - ).and_return(('repo::archive',)) - insert_execute_command_mock(('borg', 'recreate', 'repo::archive')) + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock(('borg', 'recreate', '--repo', 'repo')) module.recreate_archive( repository='repo', @@ -96,10 +98,16 @@ def test_recreate_with_remote_path(): flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.flags).should_receive( 'make_repository_archive_flags' - ).and_return(('repo::archive',)) - insert_execute_command_mock(('borg', 'recreate', '--remote-path', 'borg1', 'repo::archive')) + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock(('borg', 'recreate', '--remote-path', 'borg1', '--repo', 'repo')) module.recreate_archive( repository='repo', @@ -125,10 +133,16 @@ def test_recreate_with_lock_wait(): flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.flags).should_receive( 'make_repository_archive_flags' - ).and_return(('repo::archive',)) - insert_execute_command_mock(('borg', 'recreate', '--lock-wait', '5', 'repo::archive')) + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock(('borg', 'recreate', '--lock-wait', '5', '--repo', 'repo')) module.recreate_archive( repository='repo', @@ -153,10 +167,16 @@ def test_recreate_with_log_info(): flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.flags).should_receive( 'make_repository_archive_flags' - ).and_return(('repo::archive',)) - insert_execute_command_mock(('borg', 'recreate', '--info', 'repo::archive')) + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock(('borg', 'recreate', '--info', '--repo', 'repo')) insert_logging_mock(logging.INFO) @@ -183,10 +203,16 @@ def test_recreate_with_log_debug(): flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.flags).should_receive( 'make_repository_archive_flags' - ).and_return(('repo::archive',)) - insert_execute_command_mock(('borg', 'recreate', '--debug', '--show-rc', 'repo::archive')) + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock(('borg', 'recreate', '--debug', '--show-rc', '--repo', 'repo')) insert_logging_mock(logging.DEBUG) module.recreate_archive( @@ -212,10 +238,16 @@ def test_recreate_with_log_json(): flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.flags).should_receive( 'make_repository_archive_flags' - ).and_return(('repo::archive',)) - insert_execute_command_mock(('borg', 'recreate', '--log-json', 'repo::archive')) + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock(('borg', 'recreate', '--log-json', '--repo', 'repo')) module.recreate_archive( repository='repo', @@ -239,12 +271,18 @@ def test_recreate_with_list_filter_flags(): flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.flags).should_receive( 'make_repository_archive_flags' - ).and_return(('repo::archive',)) + ).and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module).should_receive('make_list_filter_flags').and_return('AME+-') insert_execute_command_mock( - ('borg', 'recreate', '--list', '--filter', 'AME+-', 'repo::archive') + ('borg', 'recreate', '--list', '--filter', 'AME+-', '--repo', 'repo') ) module.recreate_archive( @@ -269,13 +307,19 @@ def test_recreate_with_patterns_from_flag(): flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.flags).should_receive( 'make_repository_archive_flags' - ).and_return(('repo::archive',)) + ).and_return( + ( + '--repo', + 'repo', + ) + ) mock_patterns_file = flexmock(name='patterns_file') flexmock(module).should_receive('write_patterns_file').and_return(mock_patterns_file) insert_execute_command_mock( - ('borg', 'recreate', '--patterns-from', 'patterns_file', 'repo::archive') + ('borg', 'recreate', '--patterns-from', 'patterns_file', '--repo', 'repo') ) module.recreate_archive( @@ -300,11 +344,17 @@ def test_recreate_with_exclude_flags(): flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.flags).should_receive( 'make_repository_archive_flags' - ).and_return(('repo::archive',)) + ).and_return( + ( + '--repo', + 'repo', + ) + ) flexmock(module).should_receive('make_exclude_flags').and_return(('--exclude', 'pattern')) - insert_execute_command_mock(('borg', 'recreate', '--exclude', 'pattern', 'repo::archive')) + insert_execute_command_mock(('borg', 'recreate', '--exclude', 'pattern', '--repo', 'repo')) module.recreate_archive( repository='repo', @@ -329,10 +379,16 @@ def test_recreate_with_target_flag(): flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.flags).should_receive( 'make_repository_archive_flags' - ).and_return(('repo::archive',)) - insert_execute_command_mock(('borg', 'recreate', '--target', 'new-archive', 'repo::archive')) + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock(('borg', 'recreate', '--target', 'new-archive', '--repo', 'repo')) module.recreate_archive( repository='repo', @@ -357,11 +413,17 @@ def test_recreate_with_comment_flag(): flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.flags).should_receive( 'make_repository_archive_flags' - ).and_return(('repo::archive',)) + ).and_return( + ( + '--repo', + 'repo', + ) + ) insert_execute_command_mock( - ('borg', 'recreate', '--comment', shlex.quote('This is a test comment'), 'repo::archive') + ('borg', 'recreate', '--comment', shlex.quote('This is a test comment'), '--repo', 'repo') ) module.recreate_archive( @@ -387,11 +449,17 @@ def test_recreate_with_timestamp_flag(): flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.flags).should_receive( 'make_repository_archive_flags' - ).and_return(('repo::archive',)) + ).and_return( + ( + '--repo', + 'repo', + ) + ) insert_execute_command_mock( - ('borg', 'recreate', '--timestamp', '2023-10-01T12:00:00', 'repo::archive') + ('borg', 'recreate', '--timestamp', '2023-10-01T12:00:00', '--repo', 'repo') ) module.recreate_archive( @@ -417,10 +485,16 @@ def test_recreate_with_compression_flag(): flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.flags).should_receive( 'make_repository_archive_flags' - ).and_return(('repo::archive',)) - insert_execute_command_mock(('borg', 'recreate', '--compression', 'lz4', 'repo::archive')) + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock(('borg', 'recreate', '--compression', 'lz4', '--repo', 'repo')) module.recreate_archive( repository='repo', @@ -445,11 +519,17 @@ def test_recreate_with_chunker_params_flag(): flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.flags).should_receive( 'make_repository_archive_flags' - ).and_return(('repo::archive',)) + ).and_return( + ( + '--repo', + 'repo', + ) + ) insert_execute_command_mock( - ('borg', 'recreate', '--chunker-params', '19,23,21,4095', 'repo::archive') + ('borg', 'recreate', '--chunker-params', '19,23,21,4095', '--repo', 'repo') ) module.recreate_archive( @@ -475,10 +555,16 @@ def test_recreate_with_recompress_flag(): flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.flags).should_receive( 'make_repository_archive_flags' - ).and_return(('repo::archive',)) - insert_execute_command_mock(('borg', 'recreate', '--recompress', 'always', 'repo::archive')) + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock(('borg', 'recreate', '--recompress', 'always', '--repo', 'repo')) module.recreate_archive( repository='repo', @@ -503,10 +589,16 @@ def test_recreate_with_match_archives_star(): flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.flags).should_receive( 'make_repository_archive_flags' - ).and_return(('repo::archive',)) - insert_execute_command_mock(('borg', 'recreate', 'repo::archive')) + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock(('borg', 'recreate', '--repo', 'repo')) module.recreate_archive( repository='repo', @@ -531,10 +623,16 @@ def test_recreate_with_match_archives_regex(): flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.flags).should_receive( 'make_repository_archive_flags' - ).and_return(('repo::archive',)) - insert_execute_command_mock(('borg', 'recreate', 'repo::archive')) + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock(('borg', 'recreate', '--repo', 'repo')) module.recreate_archive( repository='repo', @@ -559,10 +657,16 @@ def test_recreate_with_match_archives_shell(): flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.flags).should_receive( 'make_repository_archive_flags' - ).and_return(('repo::archive',)) - insert_execute_command_mock(('borg', 'recreate', 'repo::archive')) + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock(('borg', 'recreate', '--repo', 'repo')) module.recreate_archive( repository='repo', @@ -582,22 +686,24 @@ def test_recreate_with_match_archives_shell(): ) -def test_recreate_with_glob_archives_flag(): +def test_recreate_with_match_archives_and_feature_available_calls_borg_with_match_archives(): flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') - flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return( - ('--glob-archives', 'foo-*') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').with_args( + 'foo-*', None, '1.2.3' + ).and_return(('--match-archives', 'foo-*')) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( + ('--repo', 'repo') ) - flexmock(module.borgmatic.borg.flags).should_receive( - 'make_repository_archive_flags' - ).and_return(('repo::archive',)) - insert_execute_command_mock(('borg', 'recreate', '--glob-archives', 'foo-*', 'repo::archive')) + flexmock(module.borgmatic.borg.flags).should_receive('make_repository_archive_flags').never() + insert_execute_command_mock(('borg', 'recreate', '--repo', 'repo', '--match-archives', 'foo-*')) module.recreate_archive( repository='repo', - archive='archive', - config={}, + archive=None, + config={'match_archives': 'foo-*'}, local_borg_version='1.2.3', recreate_arguments=flexmock( list=None, @@ -612,31 +718,93 @@ def test_recreate_with_glob_archives_flag(): ) -def test_recreate_with_match_archives_flag(): +def test_recreate_with_archives_flag_and_feature_available_calls_borg_with_match_archives(): flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') - flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return( - ('--match-archives', 'sh:foo-*') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').with_args( + 'archive', None, '1.2.3' + ).and_return(('--match-archives', 'archive')) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( + ('--repo', 'repo') ) - flexmock(module.borgmatic.borg.flags).should_receive( - 'make_repository_archive_flags' - ).and_return(('--repo', 'repo', 'archive')) + flexmock(module.borgmatic.borg.flags).should_receive('make_repository_archive_flags').never() insert_execute_command_mock( - ('borg', 'recreate', '--match-archives', 'sh:foo-*', '--repo', 'repo', 'archive') + ('borg', 'recreate', '--repo', 'repo', '--match-archives', 'archive') ) module.recreate_archive( repository='repo', archive='archive', - config={}, - local_borg_version='2.0.0b3', + config={'match_archives': 'foo-*'}, + local_borg_version='1.2.3', recreate_arguments=flexmock( list=None, target=None, comment=None, timestamp=None, - match_archives='sh:foo-*', + match_archives='foo-*', + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_match_archives_and_feature_not_available_calls_borg_without_match_archives(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').never() + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(False) + flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( + ('repo',) + ) + flexmock(module.borgmatic.borg.flags).should_receive('make_repository_archive_flags').never() + insert_execute_command_mock(('borg', 'recreate', 'repo')) + + module.recreate_archive( + repository='repo', + archive=None, + config={'match_archives': 'foo-*'}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives='foo-*', + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_archives_flags_and_feature_not_available_calls_borg_with_combined_repo_and_archive(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').never() + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(False) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) + flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').never() + insert_execute_command_mock(('borg', 'recreate', 'repo::archive')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={'match_archives': 'foo-*'}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives='foo-*', ), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', From 6a61259f1a3b1427c1d3f95fc29ae8b1cc33c49f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 1 Apr 2025 11:49:47 -0700 Subject: [PATCH 208/226] Fix a failure in the "spot" check when the archive contains a symlink (#1050). --- NEWS | 1 + borgmatic/actions/check.py | 14 ++++++---- tests/unit/actions/test_check.py | 44 ++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/NEWS b/NEWS index e320b0e8..112948af 100644 --- a/NEWS +++ b/NEWS @@ -24,6 +24,7 @@ character. * #1048: Fix a "no such file or directory" error in ZFS, Btrfs, and LVM hooks with nested directories that reside on separate devices/filesystems. + * #1050: Fix a failure in the "spot" check when the archive contains a symlink. 1.9.14 * #409: With the PagerDuty monitoring hook, send borgmatic logs to PagerDuty so they show up in the diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index b562d760..87119ca0 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -483,10 +483,12 @@ def compare_spot_check_hashes( ) source_sample_paths = tuple(random.sample(source_paths, sample_count)) working_directory = borgmatic.config.paths.get_working_directory(config) - existing_source_sample_paths = { + hashable_source_sample_path = { source_path for source_path in source_sample_paths - if os.path.exists(os.path.join(working_directory or '', source_path)) + for full_source_path in (os.path.join(working_directory or '', source_path),) + if os.path.exists(full_source_path) + if not os.path.islink(full_source_path) } logger.debug( f'Sampling {sample_count} source paths (~{spot_check_config["data_sample_percentage"]}%) for spot check' @@ -509,7 +511,7 @@ def compare_spot_check_hashes( hash_output = borgmatic.execute.execute_command_and_capture_output( (spot_check_config.get('xxh64sum_command', 'xxh64sum'),) + tuple( - path for path in source_sample_paths_subset if path in existing_source_sample_paths + path for path in source_sample_paths_subset if path in hashable_source_sample_path ), working_directory=working_directory, ) @@ -517,11 +519,13 @@ def compare_spot_check_hashes( source_hashes.update( **dict( (reversed(line.split(' ', 1)) for line in hash_output.splitlines()), - # Represent non-existent files as having empty hashes so the comparison below still works. + # Represent non-existent files as having empty hashes so the comparison below still + # works. Same thing for filesystem links, since Borg produces empty archive hashes + # for them. **{ path: '' for path in source_sample_paths_subset - if path not in existing_source_sample_paths + if path not in hashable_source_sample_path }, ) ) diff --git a/tests/unit/actions/test_check.py b/tests/unit/actions/test_check.py index 6b16a901..1ee77393 100644 --- a/tests/unit/actions/test_check.py +++ b/tests/unit/actions/test_check.py @@ -903,6 +903,7 @@ def test_compare_spot_check_hashes_returns_paths_having_failing_hashes(): None, ) flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.os.path).should_receive('islink').and_return(False) flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return( @@ -943,6 +944,7 @@ def test_compare_spot_check_hashes_returns_relative_paths_having_failing_hashes( None, ) flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.os.path).should_receive('islink').and_return(False) flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).with_args(('xxh64sum', 'foo', 'bar'), working_directory=None).and_return( @@ -983,6 +985,7 @@ def test_compare_spot_check_hashes_handles_data_sample_percentage_above_100(): None, ) flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.os.path).should_receive('islink').and_return(False) flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return( @@ -1023,6 +1026,7 @@ def test_compare_spot_check_hashes_uses_xxh64sum_command_option(): None, ) flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.os.path).should_receive('islink').and_return(False) flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).with_args(('/usr/local/bin/xxh64sum', '/foo', '/bar'), working_directory=None).and_return( @@ -1060,6 +1064,7 @@ def test_compare_spot_check_hashes_considers_path_missing_from_archive_as_not_ma None, ) flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.os.path).should_receive('islink').and_return(False) flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return( @@ -1088,6 +1093,42 @@ def test_compare_spot_check_hashes_considers_path_missing_from_archive_as_not_ma ) == ('/bar',) +def test_compare_spot_check_hashes_considers_symlink_path_as_not_matching(): + flexmock(module.random).should_receive('sample').replace_with( + lambda population, count: population[:count] + ) + flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return( + None, + ) + flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.os.path).should_receive('islink').with_args('/foo').and_return(False) + flexmock(module.os.path).should_receive('islink').with_args('/bar').and_return(True) + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).with_args(('xxh64sum', '/foo'), working_directory=None).and_return('hash1 /foo') + flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return( + ['hash1 foo', 'hash2 bar'] + ) + + assert module.compare_spot_check_hashes( + repository={'path': 'repo'}, + archive='archive', + config={ + 'checks': [ + { + 'name': 'spot', + 'data_sample_percentage': 50, + }, + ] + }, + local_borg_version=flexmock(), + global_arguments=flexmock(), + local_path=flexmock(), + remote_path=flexmock(), + source_paths=('/foo', '/bar', '/baz', '/quux'), + ) == ('/bar',) + + def test_compare_spot_check_hashes_considers_non_existent_path_as_not_matching(): flexmock(module.random).should_receive('sample').replace_with( lambda population, count: population[:count] @@ -1097,6 +1138,7 @@ def test_compare_spot_check_hashes_considers_non_existent_path_as_not_matching() ) flexmock(module.os.path).should_receive('exists').with_args('/foo').and_return(True) flexmock(module.os.path).should_receive('exists').with_args('/bar').and_return(False) + flexmock(module.os.path).should_receive('islink').and_return(False) flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).with_args(('xxh64sum', '/foo'), working_directory=None).and_return('hash1 /foo') @@ -1132,6 +1174,7 @@ def test_compare_spot_check_hashes_with_too_many_paths_feeds_them_to_commands_in None, ) flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.os.path).should_receive('islink').and_return(False) flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return( @@ -1178,6 +1221,7 @@ def test_compare_spot_check_hashes_uses_working_directory_to_access_source_paths ) flexmock(module.os.path).should_receive('exists').with_args('/working/dir/foo').and_return(True) flexmock(module.os.path).should_receive('exists').with_args('/working/dir/bar').and_return(True) + flexmock(module.os.path).should_receive('islink').and_return(False) flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).with_args(('xxh64sum', 'foo', 'bar'), working_directory='/working/dir').and_return( From e96db2e100c20556bac065f6abd76d5ac2cdf3da Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 1 Apr 2025 19:43:56 -0700 Subject: [PATCH 209/226] Fix "progress" option with the "transfer" action (#303). --- borgmatic/borg/transfer.py | 11 +++++++++-- tests/unit/borg/test_transfer.py | 5 ++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index 5e16efce..7b168035 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -32,11 +32,18 @@ def transfer_archives( + flags.make_flags('remote-path', remote_path) + flags.make_flags('umask', config.get('umask')) + flags.make_flags('log-json', global_arguments.log_json) - + flags.make_flags('lock-wait', config.get('lock_wait', None)) + + flags.make_flags('lock-wait', config.get('lock_wait')) + + flags.make_flags('progress', config.get('progress')) + ( flags.make_flags_from_arguments( transfer_arguments, - excludes=('repository', 'source_repository', 'archive', 'match_archives'), + excludes=( + 'repository', + 'source_repository', + 'archive', + 'match_archives', + 'progress', + ), ) or ( flags.make_match_archives_flags( diff --git a/tests/unit/borg/test_transfer.py b/tests/unit/borg/test_transfer.py index ced41f3c..b2b9886d 100644 --- a/tests/unit/borg/test_transfer.py +++ b/tests/unit/borg/test_transfer.py @@ -440,8 +440,11 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags').with_args('progress', True).and_return( + ('--progress',) + ) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--progress',)) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) From 017cbae4f9b3b0435b0b2e59d739f16c58a3a78f Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 1 Apr 2025 19:44:47 -0700 Subject: [PATCH 210/226] Fix for the example not showing up in generated config for empty YAML objects (#303). --- borgmatic/config/generate.py | 25 +++++++++++++++---------- tests/unit/config/test_generate.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index 5ddc26c0..734457f2 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -53,16 +53,21 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i if source_config and isinstance(source_config, list) and isinstance(source_config[0], dict): source_config = dict(collections.ChainMap(*source_config)) - config = ruamel.yaml.comments.CommentedMap( - [ - ( - field_name, - schema_to_sample_configuration( - sub_schema, (source_config or {}).get(field_name, {}), level + 1 - ), - ) - for field_name, sub_schema in borgmatic.config.schema.get_properties(schema).items() - ] + config = ( + ruamel.yaml.comments.CommentedMap( + [ + ( + field_name, + schema_to_sample_configuration( + sub_schema, (source_config or {}).get(field_name, {}), level + 1 + ), + ) + for field_name, sub_schema in borgmatic.config.schema.get_properties( + schema + ).items() + ] + ) + or example ) indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0) add_comments_to_configuration_object( diff --git a/tests/unit/config/test_generate.py b/tests/unit/config/test_generate.py index ad90bb5a..ff71b9fb 100644 --- a/tests/unit/config/test_generate.py +++ b/tests/unit/config/test_generate.py @@ -39,6 +39,35 @@ def test_schema_to_sample_configuration_generates_config_map_with_examples(): ) +def test_schema_to_sample_configuration_with_empty_object_generates_config_map_with_example(): + schema = { + 'type': 'object', + 'example': { + 'foo': 'Example 1', + 'baz': 'Example 2', + }, + } + flexmock(module.borgmatic.config.schema).should_receive('compare_types').and_return(False) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'object', {'object'} + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'string', module.SCALAR_SCHEMA_TYPES, match=all + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return({}) + flexmock(module.ruamel.yaml.comments).should_receive('CommentedMap').replace_with(dict) + flexmock(module).should_receive('add_comments_to_configuration_object') + + config = module.schema_to_sample_configuration(schema) + + assert config == dict( + [ + ('foo', 'Example 1'), + ('baz', 'Example 2'), + ] + ) + + def test_schema_to_sample_configuration_generates_config_sequence_of_strings_with_example(): flexmock(module.ruamel.yaml.comments).should_receive('CommentedSeq').replace_with(list) flexmock(module).should_receive('add_comments_to_configuration_sequence') From affe7cdc1b3e8cfa948e10d7dcee6150837a2cfc Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 1 Apr 2025 21:05:44 -0700 Subject: [PATCH 211/226] Expose propertyless YAML objects from configuration (e.g. "constants") as command-line flags (#303). --- borgmatic/commands/arguments.py | 10 ++-- tests/integration/commands/test_arguments.py | 16 ++++++ tests/unit/commands/test_arguments.py | 55 +++++++++++++++++--- 3 files changed, 69 insertions(+), 12 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 09ec0baa..a043ae6c 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -307,7 +307,7 @@ def make_argument_description(schema, flag_name): if '[0]' in flag_name: description += ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).' - if example and schema_type == 'array': + if example and schema_type in ('array', 'object'): example_buffer = io.StringIO() yaml = ruamel.yaml.YAML(typ='safe') yaml.default_flow_style = True @@ -453,13 +453,15 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names if schema_type == 'object': properties = schema.get('properties') + # If there are child properties, recurse for each one. But if there are no child properties, + # fall through so that a flag gets added below for the (empty) object. if properties: for name, child in properties.items(): add_arguments_from_schema( arguments_group, child, unparsed_arguments, names + (name,) ) - return + return # If this is an "array" type, recurse for each items type child option. Don't return yet so that # a flag also gets added below for the array itself. @@ -485,9 +487,9 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names metavar = names[-1].upper() description = make_argument_description(schema, flag_name) - # array=str instead of list here to support specifying a list as a YAML string on the + # The ...=str given here is to support specifying an object or an array as a YAML string on the # command-line. - argument_type = borgmatic.config.schema.parse_type(schema_type, array=str) + argument_type = borgmatic.config.schema.parse_type(schema_type, object=str, array=str) full_flag_name = f"--{flag_name.replace('_', '-')}" # As a UX nicety, allow boolean options that have a default of false to have command-line flags diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index a357e1b0..1a4ee87d 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -4,6 +4,22 @@ from flexmock import flexmock from borgmatic.commands import arguments as module +def test_make_argument_description_with_object_adds_example(): + assert ( + module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'object', + 'example': {'bar': 'baz'}, + }, + flag_name='flag', + ) + # Apparently different versions of ruamel.yaml serialize this + # differently. + in ('Thing. Example value: "bar: baz"' 'Thing. Example value: "{bar: baz}"') + ) + + def test_make_argument_description_with_array_adds_example(): assert ( module.make_argument_description( diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index 1e5bbe0b..bf1b4f23 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -590,6 +590,42 @@ def test_make_argument_description_without_description_bails(): ) +def test_make_argument_description_with_object_adds_example(): + buffer = flexmock() + buffer.should_receive('getvalue').and_return('{foo: example}') + flexmock(module.io).should_receive('StringIO').and_return(buffer) + yaml = flexmock() + yaml.should_receive('dump') + flexmock(module.ruamel.yaml).should_receive('YAML').and_return(yaml) + + assert ( + module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'object', + 'example': {'foo': 'example'}, + }, + flag_name='flag', + ) + == 'Thing. Example value: "{foo: example}"' + ) + + +def test_make_argument_description_with_object_skips_missing_example(): + flexmock(module.ruamel.yaml).should_receive('YAML').never() + + assert ( + module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'object', + }, + flag_name='flag', + ) + == 'Thing.' + ) + + def test_make_argument_description_with_array_adds_example(): buffer = flexmock() buffer.should_receive('getvalue').and_return('[example]') @@ -612,9 +648,7 @@ def test_make_argument_description_with_array_adds_example(): def test_make_argument_description_with_array_skips_missing_example(): - yaml = flexmock() - yaml.should_receive('dump').and_return('[example]') - flexmock(module.ruamel.yaml).should_receive('YAML').and_return(yaml) + flexmock(module.ruamel.yaml).should_receive('YAML').never() assert ( module.make_argument_description( @@ -951,12 +985,17 @@ def test_add_arguments_from_schema_with_empty_multi_type_raises(): ) -def test_add_arguments_from_schema_with_propertyless_option_does_not_add_flag(): +def test_add_arguments_from_schema_with_propertyless_option_adds_flag(): arguments_group = flexmock() - flexmock(module).should_receive('make_argument_description').never() - flexmock(module.borgmatic.config.schema).should_receive('parse_type').never() - arguments_group.should_receive('add_argument').never() - flexmock(module).should_receive('add_array_element_arguments').never() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--foo', + type=str, + metavar='FOO', + help='help', + ).once() + flexmock(module).should_receive('add_array_element_arguments') module.add_arguments_from_schema( arguments_group=arguments_group, From 4065c5d0f7bb8ca3319c4240e5d1598fc3e32aad Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 1 Apr 2025 23:04:53 -0700 Subject: [PATCH 212/226] Fix use of dashed command-line flags like "--repositories[2].append-only" generated from configuration (#303). --- borgmatic/commands/arguments.py | 35 +++++++++++++------- tests/integration/commands/test_arguments.py | 5 +-- tests/unit/commands/test_arguments.py | 32 ++++++++++++++++++ 3 files changed, 56 insertions(+), 16 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index a043ae6c..d06f6ec7 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -359,7 +359,9 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): if '[0]' not in flag_name or not unparsed_arguments or '--help' in unparsed_arguments: return - pattern = re.compile(fr'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.")}$') + pattern = re.compile( + fr'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.").replace("_", "-")}$' + ) try: # Find an existing list index flag (and its action) corresponding to the given flag name. @@ -368,7 +370,7 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): for action in arguments_group._group_actions for action_flag_name in action.option_strings if pattern.match(action_flag_name) - if f'--{flag_name}'.startswith(action_flag_name) + if f'--{flag_name.replace("_", "-")}'.startswith(action_flag_name) ) # Based on the type of the action (e.g. argparse._StoreTrueAction), look up the corresponding @@ -388,16 +390,25 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): if not pattern.match(unparsed_flag_name) or unparsed_flag_name == existing_flag_name: continue - arguments_group.add_argument( - unparsed_flag_name, - action=action_registry_name, - choices=argument_action.choices, - default=argument_action.default, - dest=unparsed_flag_name.lstrip('-'), - nargs=argument_action.nargs, - required=argument_action.nargs, - type=argument_action.type, - ) + if action_registry_name in ('store_true', 'store_false'): + arguments_group.add_argument( + unparsed_flag_name, + action=action_registry_name, + default=argument_action.default, + dest=unparsed_flag_name.lstrip('-'), + required=argument_action.nargs, + ) + else: + arguments_group.add_argument( + unparsed_flag_name, + action=action_registry_name, + choices=argument_action.choices, + default=argument_action.default, + dest=unparsed_flag_name.lstrip('-'), + nargs=argument_action.nargs, + required=argument_action.nargs, + type=argument_action.type, + ) def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names=None): diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 1a4ee87d..b3ffef0d 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -51,12 +51,9 @@ def test_add_array_element_arguments_adds_arguments_for_array_index_flags(): flexmock(arguments_group).should_receive('add_argument').with_args( '--foo[25].val', action='store_true', - choices=object, - default=object, + default=False, dest='foo[25].val', - nargs=object, required=object, - type=object, ).once() module.add_array_element_arguments( diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index bf1b4f23..138a3b2e 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -880,6 +880,38 @@ def test_add_array_element_arguments_adds_arguments_for_array_index_flags_with_e ) +def test_add_array_element_arguments_adds_arguments_for_array_index_flags_with_dashes(): + arguments_group = flexmock( + _group_actions=( + Group_action( + option_strings=('--foo[0].val-and-stuff',), + choices=flexmock(), + default=flexmock(), + nargs=flexmock(), + required=flexmock(), + type=flexmock(), + ), + ), + _registries={'action': {'store_stuff': Group_action}}, + ) + arguments_group.should_receive('add_argument').with_args( + '--foo[25].val-and-stuff', + action='store_stuff', + choices=object, + default=object, + dest='foo[25].val-and-stuff', + nargs=object, + required=object, + type=object, + ).once() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo[25].val-and-stuff', 'fooval', '--bar[1].val', 'barval'), + flag_name='foo[0].val_and_stuff', + ) + + def test_add_arguments_from_schema_with_non_dict_schema_bails(): arguments_group = flexmock() flexmock(module).should_receive('make_argument_description').never() From 7a0c56878bdc3963b4cc815111bf9ee7ede7032d Mon Sep 17 00:00:00 2001 From: Gautam Aggarwal Date: Wed, 2 Apr 2025 10:47:35 +0000 Subject: [PATCH 213/226] Applied changes --- borgmatic/config/schema.yaml | 6 +++--- borgmatic/hooks/credential/keepassxc.py | 14 ++++++++++--- tests/unit/hooks/credential/test_keepassxc.py | 20 ++++++++----------- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 15c04ee3..9c12f6c9 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -2689,10 +2689,10 @@ properties: Path to a key file for unlocking the KeePassXC database. example: /path/to/keyfile yubikey: - type: boolean + type: string description: | - Whether to use a YubiKey for unlocking the KeePassXC database. - example: true + Path or identifier for the YubiKey to use for unlocking the KeePassXC database. + example: /path/to/yubikey description: | Configuration for integration with the KeePassXC password manager. default_actions: diff --git a/borgmatic/hooks/credential/keepassxc.py b/borgmatic/hooks/credential/keepassxc.py index 1a0da572..0cb9ad12 100644 --- a/borgmatic/hooks/credential/keepassxc.py +++ b/borgmatic/hooks/credential/keepassxc.py @@ -17,7 +17,6 @@ def load_credential(hook_config, config, credential_parameters): ''' try: database_path, attribute_name = credential_parameters[:2] - extra_args = credential_parameters[2:] # Handle additional arguments like --key-file or --yubikey except ValueError: raise ValueError( f'Invalid KeePassXC credential parameters: {credential_parameters}') @@ -25,7 +24,11 @@ def load_credential(hook_config, config, credential_parameters): if not os.path.exists(expanded_database_path): raise ValueError( f'KeePassXC database path does not exist: {database_path}') - + + # Retrieve key file and Yubikey options from config + key_file = hook_config.get('key_file') + yubikey = hook_config.get('yubikey') + # Build the keepassxc-cli command command = ( tuple(shlex.split((hook_config or {}).get('keepassxc_cli_command', 'keepassxc-cli'))) @@ -37,8 +40,13 @@ def load_credential(hook_config, config, credential_parameters): expanded_database_path, attribute_name, ) - + tuple(extra_args) # Append extra arguments ) + + if key_file: + command += ('--key-file', key_file) + + if yubikey: + command += ('--yubikey', yubikey) try: return borgmatic.execute.execute_command_and_capture_output(command).rstrip(os.linesep) diff --git a/tests/unit/hooks/credential/test_keepassxc.py b/tests/unit/hooks/credential/test_keepassxc.py index c7c2306f..8b21cf08 100644 --- a/tests/unit/hooks/credential/test_keepassxc.py +++ b/tests/unit/hooks/credential/test_keepassxc.py @@ -143,9 +143,9 @@ def test_load_credential_with_key_file(): assert ( module.load_credential( - hook_config={}, + hook_config={'key_file': '/path/to/keyfile'}, config={}, - credential_parameters=('database.kdbx', 'mypassword', '--key-file', '/path/to/keyfile'), + credential_parameters=('database.kdbx', 'mypassword'), ) == 'password' ) @@ -168,6 +168,7 @@ def test_load_credential_with_yubikey(): 'database.kdbx', 'mypassword', '--yubikey', + '/path/to/yubikey', ) ).and_return( 'password' @@ -175,9 +176,9 @@ def test_load_credential_with_yubikey(): assert ( module.load_credential( - hook_config={}, + hook_config={'yubikey': '/path/to/yubikey'}, config={}, - credential_parameters=('database.kdbx', 'mypassword', '--yubikey'), + credential_parameters=('database.kdbx', 'mypassword'), ) == 'password' ) @@ -202,6 +203,7 @@ def test_load_credential_with_key_file_and_yubikey(): '--key-file', '/path/to/keyfile', '--yubikey', + '/path/to/yubikey', ) ).and_return( 'password' @@ -209,15 +211,9 @@ def test_load_credential_with_key_file_and_yubikey(): assert ( module.load_credential( - hook_config={}, + hook_config={'key_file': '/path/to/keyfile', 'yubikey': '/path/to/yubikey'}, config={}, - credential_parameters=( - 'database.kdbx', - 'mypassword', - '--key-file', - '/path/to/keyfile', - '--yubikey', - ), + credential_parameters=('database.kdbx', 'mypassword'), ) == 'password' ) From 96ec66de799191bbf09cb1ab5dc5c5578d89bd4f Mon Sep 17 00:00:00 2001 From: Gautam Aggarwal Date: Wed, 2 Apr 2025 10:50:25 +0000 Subject: [PATCH 214/226] Applied changes --- borgmatic/config/schema.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 9c12f6c9..85fcb9d0 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -2683,7 +2683,7 @@ properties: description: | Command to use instead of "keepassxc-cli". example: /usr/local/bin/keepassxc-cli - key-file: + key_file: type: string description: | Path to a key file for unlocking the KeePassXC database. From 4e555472353b0eee7e97494947f6e6d833745126 Mon Sep 17 00:00:00 2001 From: Gautam Aggarwal Date: Wed, 2 Apr 2025 15:35:12 +0000 Subject: [PATCH 215/226] Command Restructuring --- borgmatic/hooks/credential/keepassxc.py | 23 ++++--------------- tests/unit/hooks/credential/test_keepassxc.py | 14 +++++------ 2 files changed, 12 insertions(+), 25 deletions(-) diff --git a/borgmatic/hooks/credential/keepassxc.py b/borgmatic/hooks/credential/keepassxc.py index 0cb9ad12..e3799da9 100644 --- a/borgmatic/hooks/credential/keepassxc.py +++ b/borgmatic/hooks/credential/keepassxc.py @@ -25,29 +25,16 @@ def load_credential(hook_config, config, credential_parameters): if not os.path.exists(expanded_database_path): raise ValueError( f'KeePassXC database path does not exist: {database_path}') - # Retrieve key file and Yubikey options from config - key_file = hook_config.get('key_file') - yubikey = hook_config.get('yubikey') - + # Build the keepassxc-cli command command = ( tuple(shlex.split((hook_config or {}).get('keepassxc_cli_command', 'keepassxc-cli'))) - + ( - 'show', - '--show-protected', - '--attributes', - 'Password', - expanded_database_path, - attribute_name, - ) + + ('show', '--show-protected', '--attributes', 'Password') + + (('--key-file', hook_config['key_file']) if 'key_file' in hook_config else ()) + + (('--yubikey', hook_config['yubikey']) if 'yubikey' in hook_config else ()) + + (expanded_database_path, attribute_name) # Ensure database & entry are last ) - if key_file: - command += ('--key-file', key_file) - - if yubikey: - command += ('--yubikey', yubikey) - try: return borgmatic.execute.execute_command_and_capture_output(command).rstrip(os.linesep) except Exception as e: diff --git a/tests/unit/hooks/credential/test_keepassxc.py b/tests/unit/hooks/credential/test_keepassxc.py index 8b21cf08..b3d01366 100644 --- a/tests/unit/hooks/credential/test_keepassxc.py +++ b/tests/unit/hooks/credential/test_keepassxc.py @@ -132,10 +132,10 @@ def test_load_credential_with_key_file(): '--show-protected', '--attributes', 'Password', - 'database.kdbx', - 'mypassword', '--key-file', '/path/to/keyfile', + 'database.kdbx', + 'mypassword', ) ).and_return( 'password' @@ -165,10 +165,10 @@ def test_load_credential_with_yubikey(): '--show-protected', '--attributes', 'Password', - 'database.kdbx', - 'mypassword', '--yubikey', '/path/to/yubikey', + 'database.kdbx', + 'mypassword', ) ).and_return( 'password' @@ -198,12 +198,12 @@ def test_load_credential_with_key_file_and_yubikey(): '--show-protected', '--attributes', 'Password', - 'database.kdbx', - 'mypassword', '--key-file', '/path/to/keyfile', '--yubikey', '/path/to/yubikey', + 'database.kdbx', + 'mypassword', ) ).and_return( 'password' @@ -216,4 +216,4 @@ def test_load_credential_with_key_file_and_yubikey(): credential_parameters=('database.kdbx', 'mypassword'), ) == 'password' - ) + ) \ No newline at end of file From 364200c65aa2128547bb2ccfd85b4d8020f467d4 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 2 Apr 2025 09:37:52 -0700 Subject: [PATCH 216/226] Fix incorrect matching of non-zero array index flags with dashed names (#303). --- borgmatic/commands/arguments.py | 17 ++++++++--------- tests/unit/commands/test_arguments.py | 4 ++-- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index d06f6ec7..9c896c34 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -288,7 +288,7 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse ) -OMITTED_FLAG_NAMES = {'match_archives', 'progress', 'statistics', 'list_details'} +OMITTED_FLAG_NAMES = {'match-archives', 'progress', 'statistics', 'list-details'} def make_argument_description(schema, flag_name): @@ -359,9 +359,7 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): if '[0]' not in flag_name or not unparsed_arguments or '--help' in unparsed_arguments: return - pattern = re.compile( - fr'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.").replace("_", "-")}$' - ) + pattern = re.compile(fr'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.")}$') try: # Find an existing list index flag (and its action) corresponding to the given flag name. @@ -370,7 +368,7 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): for action in arguments_group._group_actions for action_flag_name in action.option_strings if pattern.match(action_flag_name) - if f'--{flag_name.replace("_", "-")}'.startswith(action_flag_name) + if f'--{flag_name}'.startswith(action_flag_name) ) # Based on the type of the action (e.g. argparse._StoreTrueAction), look up the corresponding @@ -386,6 +384,7 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): for unparsed in unparsed_arguments: unparsed_flag_name = unparsed.split('=', 1)[0] + destination_name = unparsed_flag_name.lstrip('-').replace('-', '_') if not pattern.match(unparsed_flag_name) or unparsed_flag_name == existing_flag_name: continue @@ -395,7 +394,7 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): unparsed_flag_name, action=action_registry_name, default=argument_action.default, - dest=unparsed_flag_name.lstrip('-'), + dest=destination_name, required=argument_action.nargs, ) else: @@ -404,7 +403,7 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): action=action_registry_name, choices=argument_action.choices, default=argument_action.default, - dest=unparsed_flag_name.lstrip('-'), + dest=destination_name, nargs=argument_action.nargs, required=argument_action.nargs, type=argument_action.type, @@ -488,7 +487,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names names[:-1] + (f'{names[-1]}[0]',) + (name,), ) - flag_name = '.'.join(names) + flag_name = '.'.join(names).replace('_', '-') # Certain options already have corresponding flags on individual actions (like "create # --progress"), so don't bother adding them to the global flags. @@ -501,7 +500,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names # The ...=str given here is to support specifying an object or an array as a YAML string on the # command-line. argument_type = borgmatic.config.schema.parse_type(schema_type, object=str, array=str) - full_flag_name = f"--{flag_name.replace('_', '-')}" + full_flag_name = f"--{flag_name}" # As a UX nicety, allow boolean options that have a default of false to have command-line flags # without values. diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index 138a3b2e..1f2457dc 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -899,7 +899,7 @@ def test_add_array_element_arguments_adds_arguments_for_array_index_flags_with_d action='store_stuff', choices=object, default=object, - dest='foo[25].val-and-stuff', + dest='foo[25].val_and_stuff', nargs=object, required=object, type=object, @@ -908,7 +908,7 @@ def test_add_array_element_arguments_adds_arguments_for_array_index_flags_with_d module.add_array_element_arguments( arguments_group=arguments_group, unparsed_arguments=('--foo[25].val-and-stuff', 'fooval', '--bar[1].val', 'barval'), - flag_name='foo[0].val_and_stuff', + flag_name='foo[0].val-and-stuff', ) From d5d04b89dcc8b509d898a6559bab2999972b1e13 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 2 Apr 2025 09:50:31 -0700 Subject: [PATCH 217/226] Add configuration filename to "Successfully ran configuration file" log message (#1051). --- NEWS | 1 + borgmatic/commands/borgmatic.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 112948af..817dc8b5 100644 --- a/NEWS +++ b/NEWS @@ -25,6 +25,7 @@ * #1048: Fix a "no such file or directory" error in ZFS, Btrfs, and LVM hooks with nested directories that reside on separate devices/filesystems. * #1050: Fix a failure in the "spot" check when the archive contains a symlink. + * #1051: Add configuration filename to "Successfully ran configuration file" log message. 1.9.14 * #409: With the PagerDuty monitoring hook, send borgmatic logs to PagerDuty so they show up in the diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 20ce8bca..cd0b7190 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -909,7 +909,7 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): dict( levelno=logging.INFO, levelname='INFO', - msg='Successfully ran configuration file', + msg=f'{config_filename}: Successfully ran configuration file', ) ) if results: From bbf6f2771532fd61e09db93a7fca0713d04a91ce Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 2 Apr 2025 17:08:04 -0700 Subject: [PATCH 218/226] For boolean configuration options, add separate "--foo" and "--no-foo" CLI flags (#303). --- borgmatic/commands/arguments.py | 21 +++--- borgmatic/commands/borgmatic.py | 2 +- borgmatic/config/schema.yaml | 36 +---------- borgmatic/logger.py | 9 +-- tests/unit/commands/test_arguments.py | 93 ++++++++++++--------------- tests/unit/test_logger.py | 42 +++++++----- 6 files changed, 86 insertions(+), 117 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 9c896c34..2e71ed1d 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -500,20 +500,26 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names # The ...=str given here is to support specifying an object or an array as a YAML string on the # command-line. argument_type = borgmatic.config.schema.parse_type(schema_type, object=str, array=str) - full_flag_name = f"--{flag_name}" - # As a UX nicety, allow boolean options that have a default of false to have command-line flags - # without values. - if schema_type == 'boolean' and schema.get('default') is False: + # As a UX nicety, add separate true and false flags for boolean options. + if schema_type == 'boolean': arguments_group.add_argument( - full_flag_name, + f'--{flag_name}', action='store_true', default=None, help=description, ) + no_flag_name = '.'.join(names[:-1] + ('no-' + names[-1],)).replace('_', '-') + arguments_group.add_argument( + f'--{no_flag_name}', + dest=flag_name.replace('-', '_'), + action='store_false', + default=None, + help=f'Set the --{flag_name} value to false.', + ) else: arguments_group.add_argument( - full_flag_name, + f'--{flag_name}', type=argument_type, metavar=metavar, help=description, @@ -553,9 +559,6 @@ def make_parsers(schema, unparsed_arguments): action='store_true', help='Go through the motions, but do not actually write to any repositories', ) - global_group.add_argument( - '-nc', '--no-color', dest='no_color', action='store_true', help='Disable colored output' - ) global_group.add_argument( '-v', '--verbosity', diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 097fe488..5bf41db0 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -1025,7 +1025,7 @@ def main(extra_summary_logs=[]): # pragma: no cover any_json_flags = any( getattr(sub_arguments, 'json', False) for sub_arguments in arguments.values() ) - color_enabled = should_do_markup(global_arguments.no_color or any_json_flags, configs) + color_enabled = should_do_markup(configs, any_json_flags) try: configure_logging( diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index d5c572f0..87388735 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -59,7 +59,6 @@ properties: description: | Whether the repository should be created append-only, only used for the repo-create action. Defaults to false. - default: false example: true storage_quota: type: string @@ -74,7 +73,6 @@ properties: Whether any missing parent directories of the repository path should be created, only used for the repo-create action. Defaults to false. - default: false example: true description: | A required list of local or remote repositories with paths and @@ -104,14 +102,12 @@ properties: description: | Stay in same file system; do not cross mount points beyond the given source directories. Defaults to false. - default: false example: true numeric_ids: type: boolean description: | Only store/extract numeric user and group identifiers. Defaults to false. - default: false example: true atime: type: boolean @@ -122,13 +118,11 @@ properties: ctime: type: boolean description: Store ctime into archive. Defaults to true. - default: true example: false birthtime: type: boolean description: | Store birthtime (creation date) into archive. Defaults to true. - default: true example: false read_special: type: boolean @@ -138,14 +132,12 @@ properties: used when backing up special devices such as /dev/zero. Defaults to false. But when a database hook is used, the setting here is ignored and read_special is considered true. - default: false example: true flags: type: boolean description: | Record filesystem flags (e.g. NODUMP, IMMUTABLE) in archive. Defaults to true. - default: true example: false files_cache: type: string @@ -219,7 +211,6 @@ properties: Exclude directories that contain a CACHEDIR.TAG file. See http://www.brynosaurus.com/cachedir/spec.html for details. Defaults to false. - default: false example: true exclude_if_present: type: array @@ -236,13 +227,11 @@ properties: If true, the exclude_if_present filename is included in backups. Defaults to false, meaning that the exclude_if_present filename is omitted from backups. - default: false example: true exclude_nodump: type: boolean description: | Exclude files with the NODUMP flag. Defaults to false. - default: false example: true borgmatic_source_directory: type: string @@ -274,7 +263,6 @@ properties: description: | If true, then source directories (and root pattern paths) must exist. If they don't, an error is raised. Defaults to false. - default: false example: true encryption_passcommand: type: string @@ -488,21 +476,18 @@ properties: description: | Bypass Borg error about a repository that has been moved. Defaults to false. - default: false example: true unknown_unencrypted_repo_access_is_ok: type: boolean description: | Bypass Borg error about a previously unknown unencrypted repository. Defaults to false. - default: false example: true check_i_know_what_i_am_doing: type: boolean description: | Bypass Borg confirmation about check with repair option. Defaults to false and an interactive prompt from Borg. - default: false example: true extra_borg_options: type: object @@ -828,9 +813,7 @@ properties: color: type: boolean description: | - Apply color to console output. Can be overridden with --no-color - command-line flag. Defaults to true. - default: true + Apply color to console output. Defaults to true. example: false progress: type: boolean @@ -838,7 +821,6 @@ properties: Display progress as each file or archive is processed when running supported actions. Corresponds to the "--progress" flag on those actions. Defaults to false. - default: false example: true statistics: type: boolean @@ -846,7 +828,6 @@ properties: Display statistics for an archive when running supported actions. Corresponds to the "--stats" flag on those actions. Defaults to false. - default: false example: true list_details: type: boolean @@ -854,7 +835,6 @@ properties: Display details for each file or archive as it is processed when running supported actions. Corresponds to the "--list" flag on those actions. Defaults to false. - default: false example: true skip_actions: type: array @@ -1197,7 +1177,6 @@ properties: backup itself. Defaults to true. Changing this to false prevents "borgmatic bootstrap" from extracting configuration files from the backup. - default: true example: false description: | Support for the "borgmatic bootstrap" action, used to extract @@ -1282,7 +1261,6 @@ properties: schema elements. These statements will fail unless the initial connection to the database is made by a superuser. - default: false example: true format: type: string @@ -1521,7 +1499,6 @@ properties: Use the "--add-drop-database" flag with mariadb-dump, causing the database to be dropped right before restore. Defaults to true. - default: true example: false options: type: string @@ -1669,7 +1646,6 @@ properties: Use the "--add-drop-database" flag with mysqldump, causing the database to be dropped right before restore. Defaults to true. - default: true example: false options: type: string @@ -2397,9 +2373,8 @@ properties: send_logs: type: boolean description: | - Send borgmatic logs to Apprise services as part the + Send borgmatic logs to Apprise services as part of the "finish", "fail", and "log" states. Defaults to true. - default: true example: false logs_size_limit: type: integer @@ -2509,14 +2484,12 @@ properties: description: | Verify the TLS certificate of the ping URL host. Defaults to true. - default: true example: false send_logs: type: boolean description: | - Send borgmatic logs to Healthchecks as part the "finish", + Send borgmatic logs to Healthchecks as part of the "finish", "fail", and "log" states. Defaults to true. - default: true example: false ping_body_limit: type: integer @@ -2549,7 +2522,6 @@ properties: the slug URL scheme (https://hc-ping.com// as opposed to https://hc-ping.com/). Defaults to false. - default: false example: true description: | Configuration for a monitoring integration with Healthchecks. Create @@ -2589,7 +2561,6 @@ properties: description: | Verify the TLS certificate of the push URL host. Defaults to true. - default: true example: false description: | Configuration for a monitoring integration with Uptime Kuma using @@ -2626,7 +2597,6 @@ properties: description: | Send borgmatic logs to PagerDuty when a backup errors. Defaults to true. - default: true example: false description: | Configuration for a monitoring integration with PagerDuty. Create an diff --git a/borgmatic/logger.py b/borgmatic/logger.py index 8e327b62..4eb34a34 100644 --- a/borgmatic/logger.py +++ b/borgmatic/logger.py @@ -29,12 +29,13 @@ def interactive_console(): return sys.stderr.isatty() and os.environ.get('TERM') != 'dumb' -def should_do_markup(no_color, configs): +def should_do_markup(configs, json_enabled): ''' - Given the value of the command-line no-color argument, and a dict of configuration filename to - corresponding parsed configuration, determine if we should enable color marking up. + Given a dict of configuration filename to corresponding parsed configuration (which already have + any command-line overrides applied) and whether json is enabled, determine if we should enable + color marking up. ''' - if no_color: + if json_enabled: return False if any(config.get('color', True) is False for config in configs.values()): diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index 1f2457dc..51a64761 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -1121,7 +1121,7 @@ def test_add_arguments_from_schema_with_array_and_nested_object_adds_multiple_fl ) -def test_add_arguments_from_schema_with_default_false_boolean_adds_valueless_flag(): +def test_add_arguments_from_schema_with_boolean_adds_two_valueless_flags(): arguments_group = flexmock() flexmock(module).should_receive('make_argument_description').and_return('help') flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(bool) @@ -1131,59 +1131,12 @@ def test_add_arguments_from_schema_with_default_false_boolean_adds_valueless_fla default=None, help='help', ).once() - flexmock(module).should_receive('add_array_element_arguments') - - module.add_arguments_from_schema( - arguments_group=arguments_group, - schema={ - 'type': 'object', - 'properties': { - 'foo': { - 'type': 'boolean', - 'default': False, - } - }, - }, - unparsed_arguments=(), - ) - - -def test_add_arguments_from_schema_with_default_true_boolean_adds_value_flag(): - arguments_group = flexmock() - flexmock(module).should_receive('make_argument_description').and_return('help') - flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(bool) arguments_group.should_receive('add_argument').with_args( - '--foo', - type=bool, - metavar='FOO', - help='help', - ).once() - flexmock(module).should_receive('add_array_element_arguments') - - module.add_arguments_from_schema( - arguments_group=arguments_group, - schema={ - 'type': 'object', - 'properties': { - 'foo': { - 'type': 'boolean', - 'default': True, - } - }, - }, - unparsed_arguments=(), - ) - - -def test_add_arguments_from_schema_with_defaultless_boolean_adds_value_flag(): - arguments_group = flexmock() - flexmock(module).should_receive('make_argument_description').and_return('help') - flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(bool) - arguments_group.should_receive('add_argument').with_args( - '--foo', - type=bool, - metavar='FOO', - help='help', + '--no-foo', + dest='foo', + action='store_false', + default=None, + help=object, ).once() flexmock(module).should_receive('add_array_element_arguments') @@ -1201,6 +1154,40 @@ def test_add_arguments_from_schema_with_defaultless_boolean_adds_value_flag(): ) +def test_add_arguments_from_schema_with_nested_boolean_adds_two_valueless_flags(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(bool) + arguments_group.should_receive('add_argument').with_args( + '--foo.bar.baz-quux', + action='store_true', + default=None, + help='help', + ).once() + arguments_group.should_receive('add_argument').with_args( + '--foo.bar.no-baz-quux', + dest='foo.bar.baz_quux', + action='store_false', + default=None, + help=object, + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'baz_quux': { + 'type': 'boolean', + } + }, + }, + unparsed_arguments=(), + names=('foo', 'bar'), + ) + + def test_add_arguments_from_schema_skips_omitted_flag_name(): arguments_group = flexmock() flexmock(module).should_receive('make_argument_description').and_return('help') diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index dffaafc0..d179735f 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -44,19 +44,23 @@ def test_interactive_console_true_when_isatty_and_TERM_is_not_dumb(capsys): assert module.interactive_console() is True -def test_should_do_markup_respects_no_color_value(): - flexmock(module.os.environ).should_receive('get').and_return(None) +def test_should_do_markup_respects_json_enabled_value(): + flexmock(module.os.environ).should_receive('get').never() flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=True, configs={}) is False + assert module.should_do_markup(configs={}, json_enabled=True) is False def test_should_do_markup_respects_config_value(): flexmock(module.os.environ).should_receive('get').and_return(None) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=False, configs={'foo.yaml': {'color': False}}) is False + assert ( + module.should_do_markup(configs={'foo.yaml': {'color': False}}, json_enabled=False) is False + ) flexmock(module).should_receive('interactive_console').and_return(True).once() - assert module.should_do_markup(no_color=False, configs={'foo.yaml': {'color': True}}) is True + assert ( + module.should_do_markup(configs={'foo.yaml': {'color': True}}, json_enabled=False) is True + ) def test_should_do_markup_prefers_any_false_config_value(): @@ -65,11 +69,11 @@ def test_should_do_markup_prefers_any_false_config_value(): assert ( module.should_do_markup( - no_color=False, configs={ 'foo.yaml': {'color': True}, 'bar.yaml': {'color': False}, }, + json_enabled=False, ) is False ) @@ -83,14 +87,16 @@ def test_should_do_markup_respects_PY_COLORS_environment_variable(): flexmock(module).should_receive('to_bool').and_return(True) - assert module.should_do_markup(no_color=False, configs={}) is True + assert module.should_do_markup(configs={}, json_enabled=False) is True -def test_should_do_markup_prefers_no_color_value_to_config_value(): +def test_should_do_markup_prefers_json_enabled_value_to_config_value(): flexmock(module.os.environ).should_receive('get').and_return(None) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=True, configs={'foo.yaml': {'color': True}}) is False + assert ( + module.should_do_markup(configs={'foo.yaml': {'color': True}}, json_enabled=True) is False + ) def test_should_do_markup_prefers_config_value_to_environment_variables(): @@ -98,7 +104,9 @@ def test_should_do_markup_prefers_config_value_to_environment_variables(): flexmock(module).should_receive('to_bool').and_return(True) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=False, configs={'foo.yaml': {'color': False}}) is False + assert ( + module.should_do_markup(configs={'foo.yaml': {'color': False}}, json_enabled=False) is False + ) def test_should_do_markup_prefers_no_color_value_to_environment_variables(): @@ -106,14 +114,14 @@ def test_should_do_markup_prefers_no_color_value_to_environment_variables(): flexmock(module).should_receive('to_bool').and_return(True) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=True, configs={}) is False + assert module.should_do_markup(configs={}, json_enabled=False) is False def test_should_do_markup_respects_interactive_console_value(): flexmock(module.os.environ).should_receive('get').and_return(None) flexmock(module).should_receive('interactive_console').and_return(True) - assert module.should_do_markup(no_color=False, configs={}) is True + assert module.should_do_markup(configs={}, json_enabled=False) is True def test_should_do_markup_prefers_PY_COLORS_to_interactive_console_value(): @@ -124,7 +132,7 @@ def test_should_do_markup_prefers_PY_COLORS_to_interactive_console_value(): flexmock(module).should_receive('to_bool').and_return(True) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=False, configs={}) is True + assert module.should_do_markup(configs={}, json_enabled=False) is True def test_should_do_markup_prefers_NO_COLOR_to_interactive_console_value(): @@ -132,7 +140,7 @@ def test_should_do_markup_prefers_NO_COLOR_to_interactive_console_value(): flexmock(module.os.environ).should_receive('get').with_args('NO_COLOR', None).and_return('True') flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=False, configs={}) is False + assert module.should_do_markup(configs={}, json_enabled=False) is False def test_should_do_markup_respects_NO_COLOR_environment_variable(): @@ -140,7 +148,7 @@ def test_should_do_markup_respects_NO_COLOR_environment_variable(): flexmock(module.os.environ).should_receive('get').with_args('PY_COLORS', None).and_return(None) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=False, configs={}) is False + assert module.should_do_markup(configs={}, json_enabled=False) is False def test_should_do_markup_ignores_empty_NO_COLOR_environment_variable(): @@ -148,7 +156,7 @@ def test_should_do_markup_ignores_empty_NO_COLOR_environment_variable(): flexmock(module.os.environ).should_receive('get').with_args('PY_COLORS', None).and_return(None) flexmock(module).should_receive('interactive_console').and_return(True) - assert module.should_do_markup(no_color=False, configs={}) is True + assert module.should_do_markup(configs={}, json_enabled=False) is True def test_should_do_markup_prefers_NO_COLOR_to_PY_COLORS(): @@ -160,7 +168,7 @@ def test_should_do_markup_prefers_NO_COLOR_to_PY_COLORS(): ) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=False, configs={}) is False + assert module.should_do_markup(configs={}, json_enabled=False) is False def test_multi_stream_handler_logs_to_handler_for_log_level(): From d2c3ed26a90877c79fca6756ae9bf0b0a3c3ac26 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 2 Apr 2025 23:15:21 -0700 Subject: [PATCH 219/226] Make a CLI flag for any config option that's a list of scalars (#303). --- borgmatic/commands/arguments.py | 26 +++++---- borgmatic/config/arguments.py | 16 +++--- tests/unit/commands/test_arguments.py | 79 +++++++++++++++++++++------ tests/unit/config/test_arguments.py | 7 +++ 4 files changed, 93 insertions(+), 35 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 2e71ed1d..c2ebbd88 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -300,12 +300,12 @@ def make_argument_description(schema, flag_name): description = schema.get('description') schema_type = schema.get('type') example = schema.get('example') - - if not description: - return None + pieces = [description] if description else [] if '[0]' in flag_name: - description += ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).' + pieces.append( + ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).' + ) if example and schema_type in ('array', 'object'): example_buffer = io.StringIO() @@ -313,11 +313,9 @@ def make_argument_description(schema, flag_name): yaml.default_flow_style = True yaml.dump(example, example_buffer) - description += f' Example value: "{example_buffer.getvalue().strip()}"' + pieces.append(f'Example value: "{example_buffer.getvalue().strip()}"') - description = description.replace('%', '%%') - - return description + return ' '.join(pieces).replace('%', '%%') def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): @@ -476,7 +474,8 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names # If this is an "array" type, recurse for each items type child option. Don't return yet so that # a flag also gets added below for the array itself. if schema_type == 'array': - properties = borgmatic.config.schema.get_properties(schema.get('items', {})) + items = schema.get('items', {}) + properties = borgmatic.config.schema.get_properties(items) if properties: for name, child in properties.items(): @@ -486,6 +485,11 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names unparsed_arguments, names[:-1] + (f'{names[-1]}[0]',) + (name,), ) + # If there aren't any children, then this is an array of scalars. Recurse accordingly. + else: + add_arguments_from_schema( + arguments_group, items, unparsed_arguments, names[:-1] + (f'{names[-1]}[0]',) + ) flag_name = '.'.join(names).replace('_', '-') @@ -497,8 +501,8 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names metavar = names[-1].upper() description = make_argument_description(schema, flag_name) - # The ...=str given here is to support specifying an object or an array as a YAML string on the - # command-line. + # The object=str and array=str given here is to support specifying an object or an array as a + # YAML string on the command-line. argument_type = borgmatic.config.schema.parse_type(schema_type, object=str, array=str) # As a UX nicety, add separate true and false flags for boolean options. diff --git a/borgmatic/config/arguments.py b/borgmatic/config/arguments.py index 3252d779..e23587d7 100644 --- a/borgmatic/config/arguments.py +++ b/borgmatic/config/arguments.py @@ -36,15 +36,15 @@ def set_values(config, keys, value): list_key = match.group('list_name') list_index = int(match.group('index')) - if len(keys) == 1: - config[list_key][list_index] = value - - return - - if list_key not in config: - config[list_key] = [] - try: + if len(keys) == 1: + config[list_key][list_index] = value + + return + + if list_key not in config: + config[list_key] = [] + set_values(config[list_key][list_index], keys[1:], value) except IndexError: raise ValueError(f'Argument list index {first_key} is out of range') diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index 51a64761..efc437b3 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -577,19 +577,6 @@ def test_parse_arguments_for_actions_raises_error_when_no_action_is_specified(): module.parse_arguments_for_actions(('config',), action_parsers, global_parser) -def test_make_argument_description_without_description_bails(): - assert ( - module.make_argument_description( - schema={ - 'description': None, - 'type': 'not yours', - }, - flag_name='flag', - ) - is None - ) - - def test_make_argument_description_with_object_adds_example(): buffer = flexmock() buffer.should_receive('getvalue').and_return('{foo: example}') @@ -611,6 +598,26 @@ def test_make_argument_description_with_object_adds_example(): ) +def test_make_argument_description_without_description_and_with_object_sets_example(): + buffer = flexmock() + buffer.should_receive('getvalue').and_return('{foo: example}') + flexmock(module.io).should_receive('StringIO').and_return(buffer) + yaml = flexmock() + yaml.should_receive('dump') + flexmock(module.ruamel.yaml).should_receive('YAML').and_return(yaml) + + assert ( + module.make_argument_description( + schema={ + 'type': 'object', + 'example': {'foo': 'example'}, + }, + flag_name='flag', + ) + == 'Example value: "{foo: example}"' + ) + + def test_make_argument_description_with_object_skips_missing_example(): flexmock(module.ruamel.yaml).should_receive('YAML').never() @@ -647,6 +654,26 @@ def test_make_argument_description_with_array_adds_example(): ) +def test_make_argument_description_without_description_and_with_array_sets_example(): + buffer = flexmock() + buffer.should_receive('getvalue').and_return('[example]') + flexmock(module.io).should_receive('StringIO').and_return(buffer) + yaml = flexmock() + yaml.should_receive('dump') + flexmock(module.ruamel.yaml).should_receive('YAML').and_return(yaml) + + assert ( + module.make_argument_description( + schema={ + 'type': 'array', + 'example': ['example'], + }, + flag_name='flag', + ) + == 'Example value: "[example]"' + ) + + def test_make_argument_description_with_array_skips_missing_example(): flexmock(module.ruamel.yaml).should_receive('YAML').never() @@ -672,6 +699,15 @@ def test_make_argument_description_with_array_index_in_flag_name_adds_to_descrip ) +def test_make_argument_description_without_description_and_with_array_index_in_flag_name_sets_description(): + assert 'list element' in module.make_argument_description( + schema={ + 'type': 'something', + }, + flag_name='flag[0]', + ) + + def test_make_argument_description_escapes_percent_character(): assert ( module.make_argument_description( @@ -1043,10 +1079,21 @@ def test_add_arguments_from_schema_with_propertyless_option_adds_flag(): ) -def test_add_arguments_from_schema_with_array_adds_flag(): +def test_add_arguments_from_schema_with_array_of_scalars_adds_multiple_flags(): arguments_group = flexmock() flexmock(module).should_receive('make_argument_description').and_return('help') - flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(str) + flexmock(module.borgmatic.config.schema).should_receive('parse_type').with_args( + 'integer', object=str, array=str + ).and_return(int) + flexmock(module.borgmatic.config.schema).should_receive('parse_type').with_args( + 'array', object=str, array=str + ).and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--foo[0]', + type=int, + metavar='FOO[0]', + help='help', + ).once() arguments_group.should_receive('add_argument').with_args( '--foo', type=str, @@ -1072,7 +1119,7 @@ def test_add_arguments_from_schema_with_array_adds_flag(): ) -def test_add_arguments_from_schema_with_array_and_nested_object_adds_multiple_flags(): +def test_add_arguments_from_schema_with_array_of_objects_adds_multiple_flags(): arguments_group = flexmock() flexmock(module).should_receive('make_argument_description').and_return('help 1').and_return( 'help 2' diff --git a/tests/unit/config/test_arguments.py b/tests/unit/config/test_arguments.py index 2c07f815..79f0236a 100644 --- a/tests/unit/config/test_arguments.py +++ b/tests/unit/config/test_arguments.py @@ -50,6 +50,13 @@ def test_set_values_with_list_index_key_out_of_range_raises(): module.set_values(config=config, keys=('foo', 'bar[1]', 'baz'), value=5) +def test_set_values_with_final_list_index_key_out_of_range_raises(): + config = {'foo': {'bar': [{'option': 'value'}]}} + + with pytest.raises(ValueError): + module.set_values(config=config, keys=('foo', 'bar[1]'), value=5) + + def test_set_values_with_list_index_key_missing_list_and_out_of_range_raises(): config = {'other': 'value'} From d0a5aa63be3e0b7538a9580dee6440ba26f5c6d3 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 3 Apr 2025 09:24:47 -0700 Subject: [PATCH 220/226] Add a TL;DR to NEWS since 2.0.0 is such a huge release and ain't nobody got time for reading a huge changelog. --- NEWS | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 723ba5dc..87c1db3e 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,7 @@ 2.0.0.dev0 + * TL;DR: More flexible, completely revamped command hooks. All config options settable on the + command-line. Config option defaults for many command-line flags. New "key import" and "recreate" + actions. Almost everything is backwards compatible. * #262: Add a "default_actions" option that supports disabling default actions when borgmatic is run without any command-line arguments. * #303: Deprecate the "--override" flag in favor of direct command-line flags for every borgmatic @@ -32,7 +35,7 @@ * #1048: Fix a "no such file or directory" error in ZFS, Btrfs, and LVM hooks with nested directories that reside on separate devices/filesystems. * #1050: Fix a failure in the "spot" check when the archive contains a symlink. - * #1051: Add configuration filename to "Successfully ran configuration file" log message. + * #1051: Add configuration filename to the "Successfully ran configuration file" log message. 1.9.14 * #409: With the PagerDuty monitoring hook, send borgmatic logs to PagerDuty so they show up in the From 248999c23effc077d703f203880994030a1193b4 Mon Sep 17 00:00:00 2001 From: Gautam Aggarwal Date: Thu, 3 Apr 2025 17:10:52 +0000 Subject: [PATCH 221/226] Final --- borgmatic/config/schema.yaml | 8 ++++++-- borgmatic/hooks/credential/keepassxc.py | 2 +- tests/unit/hooks/credential/test_keepassxc.py | 8 ++++---- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 85fcb9d0..991b072d 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -2691,8 +2691,12 @@ properties: yubikey: type: string description: | - Path or identifier for the YubiKey to use for unlocking the KeePassXC database. - example: /path/to/yubikey + YubiKey slot and optional serial number used to access the KeePassXC database. + Format: "", where: + - is the YubiKey slot number (e.g., `1` or `2`). + - (optional) is the YubiKey's serial number (e.g., `1:7370001`). + example: "1:7370001" + description: | Configuration for integration with the KeePassXC password manager. default_actions: diff --git a/borgmatic/hooks/credential/keepassxc.py b/borgmatic/hooks/credential/keepassxc.py index e3799da9..acaeb449 100644 --- a/borgmatic/hooks/credential/keepassxc.py +++ b/borgmatic/hooks/credential/keepassxc.py @@ -16,7 +16,7 @@ def load_credential(hook_config, config, credential_parameters): Raise ValueError if keepassxc-cli can't retrieve the credential. ''' try: - database_path, attribute_name = credential_parameters[:2] + (database_path, attribute_name) = credential_parameters except ValueError: raise ValueError( f'Invalid KeePassXC credential parameters: {credential_parameters}') diff --git a/tests/unit/hooks/credential/test_keepassxc.py b/tests/unit/hooks/credential/test_keepassxc.py index b3d01366..d34b7232 100644 --- a/tests/unit/hooks/credential/test_keepassxc.py +++ b/tests/unit/hooks/credential/test_keepassxc.py @@ -166,7 +166,7 @@ def test_load_credential_with_yubikey(): '--attributes', 'Password', '--yubikey', - '/path/to/yubikey', + '1:7370001', 'database.kdbx', 'mypassword', ) @@ -176,7 +176,7 @@ def test_load_credential_with_yubikey(): assert ( module.load_credential( - hook_config={'yubikey': '/path/to/yubikey'}, + hook_config={'yubikey': '1:7370001'}, config={}, credential_parameters=('database.kdbx', 'mypassword'), ) @@ -201,7 +201,7 @@ def test_load_credential_with_key_file_and_yubikey(): '--key-file', '/path/to/keyfile', '--yubikey', - '/path/to/yubikey', + '2', 'database.kdbx', 'mypassword', ) @@ -211,7 +211,7 @@ def test_load_credential_with_key_file_and_yubikey(): assert ( module.load_credential( - hook_config={'key_file': '/path/to/keyfile', 'yubikey': '/path/to/yubikey'}, + hook_config={'key_file': '/path/to/keyfile', 'yubikey': '2'}, config={}, credential_parameters=('database.kdbx', 'mypassword'), ) From 9407f246748114fd02fa92c968efd62b02bd2768 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 3 Apr 2025 11:28:32 -0700 Subject: [PATCH 222/226] Fix setting of "--checks" on the command-line (#303). --- borgmatic/actions/check.py | 2 +- borgmatic/config/arguments.py | 7 ++++--- tests/unit/config/test_arguments.py | 28 ++++++++++++++++++++++++++-- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index 1e65eecb..13023ff9 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -170,7 +170,7 @@ def filter_checks_on_frequency( if calendar.day_name[datetime_now().weekday()] not in days: logger.info( - f"Skipping {check} check due to day of the week; check only runs on {'/'.join(days)} (use --force to check anyway)" + f"Skipping {check} check due to day of the week; check only runs on {'/'.join(day.title() for day in days)} (use --force to check anyway)" ) filtered_checks.remove(check) continue diff --git a/borgmatic/config/arguments.py b/borgmatic/config/arguments.py index e23587d7..d5996f0a 100644 --- a/borgmatic/config/arguments.py +++ b/borgmatic/config/arguments.py @@ -46,7 +46,7 @@ def set_values(config, keys, value): config[list_key] = [] set_values(config[list_key][list_index], keys[1:], value) - except IndexError: + except (IndexError, KeyError): raise ValueError(f'Argument list index {first_key} is out of range') return @@ -75,12 +75,13 @@ def type_for_option(schema, option_keys): for key in option_keys: # Support "name[0]"-style list index syntax. match = LIST_INDEX_KEY_PATTERN.match(key) + properties = borgmatic.config.schema.get_properties(option_schema) try: if match: - option_schema = option_schema['properties'][match.group('list_name')]['items'] + option_schema = properties[match.group('list_name')]['items'] else: - option_schema = option_schema['properties'][key] + option_schema = properties[key] except KeyError: return None diff --git a/tests/unit/config/test_arguments.py b/tests/unit/config/test_arguments.py index 79f0236a..fdf5f0f5 100644 --- a/tests/unit/config/test_arguments.py +++ b/tests/unit/config/test_arguments.py @@ -73,6 +73,10 @@ def test_set_values_with_final_list_index_key_adds_it_to_config(): def test_type_for_option_with_option_finds_type(): + flexmock(module.borgmatic.config.schema).should_receive('get_properties').replace_with( + lambda sub_schema: sub_schema['properties'] + ) + assert ( module.type_for_option( schema={'type': 'object', 'properties': {'foo': {'type': 'integer'}}}, @@ -83,6 +87,10 @@ def test_type_for_option_with_option_finds_type(): def test_type_for_option_with_nested_option_finds_type(): + flexmock(module.borgmatic.config.schema).should_receive('get_properties').replace_with( + lambda sub_schema: sub_schema['properties'] + ) + assert ( module.type_for_option( schema={ @@ -98,6 +106,10 @@ def test_type_for_option_with_nested_option_finds_type(): def test_type_for_option_with_missing_nested_option_finds_nothing(): + flexmock(module.borgmatic.config.schema).should_receive('get_properties').replace_with( + lambda sub_schema: sub_schema['properties'] + ) + assert ( module.type_for_option( schema={ @@ -113,6 +125,10 @@ def test_type_for_option_with_missing_nested_option_finds_nothing(): def test_type_for_option_with_typeless_nested_option_finds_nothing(): + flexmock(module.borgmatic.config.schema).should_receive('get_properties').replace_with( + lambda sub_schema: sub_schema['properties'] + ) + assert ( module.type_for_option( schema={ @@ -125,7 +141,11 @@ def test_type_for_option_with_typeless_nested_option_finds_nothing(): ) -def test_type_for_list_index_option_finds_type(): +def test_type_for_option_with_list_index_option_finds_type(): + flexmock(module.borgmatic.config.schema).should_receive('get_properties').replace_with( + lambda sub_schema: sub_schema['properties'] + ) + assert ( module.type_for_option( schema={ @@ -138,7 +158,11 @@ def test_type_for_list_index_option_finds_type(): ) -def test_type_for_nested_list_index_option_finds_type(): +def test_type_for_option_with_nested_list_index_option_finds_type(): + flexmock(module.borgmatic.config.schema).should_receive('get_properties').replace_with( + lambda sub_schema: sub_schema['properties'] + ) + assert ( module.type_for_option( schema={ From e8542f361388b635378fc6cee225346ea636a7b2 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 3 Apr 2025 11:41:58 -0700 Subject: [PATCH 223/226] Fix KeePassXC error when "keepassxc:" option is not present, add new options to NEWS (#1047). --- NEWS | 1 + borgmatic/config/schema.yaml | 10 +++--- borgmatic/hooks/credential/keepassxc.py | 32 +++++++++++-------- tests/unit/hooks/credential/test_keepassxc.py | 4 +-- 4 files changed, 26 insertions(+), 21 deletions(-) diff --git a/NEWS b/NEWS index 817dc8b5..7cbaa679 100644 --- a/NEWS +++ b/NEWS @@ -22,6 +22,7 @@ "working_directory" are used. * #1044: Fix an error in the systemd credential hook when the credential name contains a "." character. + * #1047: Add "key-file" and "yubikey" options to the KeePassXC credential hook. * #1048: Fix a "no such file or directory" error in ZFS, Btrfs, and LVM hooks with nested directories that reside on separate devices/filesystems. * #1050: Fix a failure in the "spot" check when the archive contains a symlink. diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 991b072d..98923e90 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -2691,12 +2691,12 @@ properties: yubikey: type: string description: | - YubiKey slot and optional serial number used to access the KeePassXC database. - Format: "", where: - - is the YubiKey slot number (e.g., `1` or `2`). - - (optional) is the YubiKey's serial number (e.g., `1:7370001`). + YubiKey slot and optional serial number used to access the + KeePassXC database. The format is "", where: + * is the YubiKey slot number (e.g., `1` or `2`). + * (optional) is the YubiKey's serial number (e.g., + `7370001`). example: "1:7370001" - description: | Configuration for integration with the KeePassXC password manager. default_actions: diff --git a/borgmatic/hooks/credential/keepassxc.py b/borgmatic/hooks/credential/keepassxc.py index acaeb449..c3605fcf 100644 --- a/borgmatic/hooks/credential/keepassxc.py +++ b/borgmatic/hooks/credential/keepassxc.py @@ -18,24 +18,28 @@ def load_credential(hook_config, config, credential_parameters): try: (database_path, attribute_name) = credential_parameters except ValueError: - raise ValueError( f'Invalid KeePassXC credential parameters: {credential_parameters}') + raise ValueError(f'Invalid KeePassXC credential parameters: {credential_parameters}') expanded_database_path = os.path.expanduser(database_path) if not os.path.exists(expanded_database_path): - raise ValueError( f'KeePassXC database path does not exist: {database_path}') - - - # Build the keepassxc-cli command + raise ValueError(f'KeePassXC database path does not exist: {database_path}') + + # Build the keepassxc-cli command. command = ( tuple(shlex.split((hook_config or {}).get('keepassxc_cli_command', 'keepassxc-cli'))) - + ('show', '--show-protected', '--attributes', 'Password') - + (('--key-file', hook_config['key_file']) if 'key_file' in hook_config else ()) - + (('--yubikey', hook_config['yubikey']) if 'yubikey' in hook_config else ()) - + (expanded_database_path, attribute_name) # Ensure database & entry are last + + ('show', '--show-protected', '--attributes', 'Password') + + ( + ('--key-file', hook_config['key_file']) + if hook_config and hook_config.get('key_file') + else () + ) + + ( + ('--yubikey', hook_config['yubikey']) + if hook_config and hook_config.get('yubikey') + else () + ) + + (expanded_database_path, attribute_name) # Ensure database and entry are last. ) - - try: - return borgmatic.execute.execute_command_and_capture_output(command).rstrip(os.linesep) - except Exception as e: - raise ValueError(f'Failed to retrieve credential: {e}') + + return borgmatic.execute.execute_command_and_capture_output(command).rstrip(os.linesep) diff --git a/tests/unit/hooks/credential/test_keepassxc.py b/tests/unit/hooks/credential/test_keepassxc.py index d34b7232..ccb9173b 100644 --- a/tests/unit/hooks/credential/test_keepassxc.py +++ b/tests/unit/hooks/credential/test_keepassxc.py @@ -135,7 +135,7 @@ def test_load_credential_with_key_file(): '--key-file', '/path/to/keyfile', 'database.kdbx', - 'mypassword', + 'mypassword', ) ).and_return( 'password' @@ -216,4 +216,4 @@ def test_load_credential_with_key_file_and_yubikey(): credential_parameters=('database.kdbx', 'mypassword'), ) == 'password' - ) \ No newline at end of file + ) From 09212961a43d28add13fadf3f53a1e2c76758be4 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 3 Apr 2025 12:55:26 -0700 Subject: [PATCH 224/226] Add action "--help" note about running compact after recreate (#1053). --- borgmatic/commands/arguments.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 671893c4..15bb286c 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1525,8 +1525,8 @@ def make_parsers(): recreate_parser = action_parsers.add_parser( 'recreate', aliases=ACTION_ALIASES['recreate'], - help='Recreate an archive in a repository', - description='Recreate an archive in a repository', + help='Recreate an archive in a repository (with Borg 1.2+, you must run compact afterwards to actually free space)', + description='Recreate an archive in a repository (with Borg 1.2+, you must run compact afterwards to actually free space)', add_help=False, ) recreate_group = recreate_parser.add_argument_group('recreate arguments') From 3eabda45f2880c348a7f2ae0076729102f625b70 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 3 Apr 2025 16:21:22 -0700 Subject: [PATCH 225/226] If a boolean option name already starts with "no_", don't add a "--no-no-..." CLI flag (#303). --- borgmatic/commands/arguments.py | 7 +++++- tests/unit/commands/test_arguments.py | 33 +++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index c2ebbd88..0cdb3357 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -513,7 +513,12 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names default=None, help=description, ) - no_flag_name = '.'.join(names[:-1] + ('no-' + names[-1],)).replace('_', '-') + + if names[-1].startswith('no_'): + no_flag_name = '.'.join(names[:-1] + (names[-1][len('no_') :],)).replace('_', '-') + else: + no_flag_name = '.'.join(names[:-1] + ('no-' + names[-1],)).replace('_', '-') + arguments_group.add_argument( f'--{no_flag_name}', dest=flag_name.replace('-', '_'), diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index efc437b3..f3094277 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -1235,6 +1235,39 @@ def test_add_arguments_from_schema_with_nested_boolean_adds_two_valueless_flags( ) +def test_add_arguments_from_schema_with_boolean_with_name_prefixed_with_no_adds_two_valueless_flags_and_removes_the_no_for_one(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(bool) + arguments_group.should_receive('add_argument').with_args( + '--no-foo', + action='store_true', + default=None, + help='help', + ).once() + arguments_group.should_receive('add_argument').with_args( + '--foo', + dest='no_foo', + action='store_false', + default=None, + help=object, + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'no_foo': { + 'type': 'boolean', + } + }, + }, + unparsed_arguments=(), + ) + + def test_add_arguments_from_schema_skips_omitted_flag_name(): arguments_group = flexmock() flexmock(module).should_receive('make_argument_description').and_return('help') From 9ea55d9aa31b82319571f3a88226788d0e643fb4 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 3 Apr 2025 16:38:17 -0700 Subject: [PATCH 226/226] Add a documentation note about a limitation: You can't pass flags as values to flags (#303). --- docs/how-to/make-per-application-backups.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index db381999..a81cd943 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -538,6 +538,12 @@ the `--list` flag that's only present on particular actions. Similarly with `progress` and `--progress`, `statistics` and `--stats`, and `match_archives` and `--match-archives`. +Also note that if you want to pass a command-line flag itself as a value to one +of these override flags, that may not work. For instance, specifying +`--extra-borg-options.create --no-cache-sync` results in an error, because +`--no-cache-sync` gets interpreted as a borgmatic option (which in this case +doesn't exist) rather than a Borg option. + An alternate to command-line overrides is passing in your values via [environment variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/).