diff --git a/NEWS b/NEWS index 0b96a621..b2a83ec7 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,95 @@ -1.9.10.dev0 +2.0.0.dev0 + * TL;DR: More flexible, completely revamped command hooks. All config options settable on the + command-line. Config option defaults for many command-line flags. New "key import" and "recreate" + actions. Almost everything is backwards compatible. + * #262: Add a "default_actions" option that supports disabling default actions when borgmatic is + run without any command-line arguments. + * #303: Deprecate the "--override" flag in favor of direct command-line flags for every borgmatic + configuration option. See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides + * #303: Add configuration options that serve as defaults for some (but not all) command-line + action flags. For example, each entry in "repositories:" now has an "encryption" option that + applies to the "repo-create" action, serving as a default for the "--encryption" flag. See the + documentation for more information: https://torsion.org/borgmatic/docs/reference/configuration/ + * #345: Add a "key import" action to import a repository key from backup. + * #422: Add home directory expansion to file-based and KeePassXC credential hooks. + * #610: Add a "recreate" action for recreating archives, for instance for retroactively excluding + particular files from existing archives. + * #790, #821: Deprecate all "before_*", "after_*" and "on_error" command hooks in favor of more + flexible "commands:". See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/ + * #790: BREAKING: For both new and deprecated command hooks, run a configured "after" hook even if + an error occurs first. This allows you to perform cleanup steps that correspond to "before" + preparation commands—even when something goes wrong. + * #790: BREAKING: Run all command hooks (both new and deprecated) respecting the + "working_directory" option if configured, meaning that hook commands are run in that directory. + * #836: Add a custom command option for the SQLite hook. + * #837: Add custom command options for the MongoDB hook. + * #1010: When using Borg 2, don't pass the "--stats" flag to "borg prune". + * #1020: Document a database use case involving a temporary database client container: + https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers + * #1037: Fix an error with the "extract" action when both a remote repository and a + "working_directory" are used. + * #1044: Fix an error in the systemd credential hook when the credential name contains a "." + character. + * #1047: Add "key-file" and "yubikey" options to the KeePassXC credential hook. + * #1048: Fix a "no such file or directory" error in ZFS, Btrfs, and LVM hooks with nested + directories that reside on separate devices/filesystems. + * #1050: Fix a failure in the "spot" check when the archive contains a symlink. + * #1051: Add configuration filename to the "Successfully ran configuration file" log message. + +1.9.14 + * #409: With the PagerDuty monitoring hook, send borgmatic logs to PagerDuty so they show up in the + incident UI. See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook + * #936: Clarify Zabbix monitoring hook documentation about creating items: + https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#zabbix-hook + * #1017: Fix a regression in which some MariaDB/MySQL passwords were not escaped correctly. + * #1021: Fix a regression in which the "exclude_patterns" option didn't expand "~" (the user's + home directory). This fix means that all "patterns" and "patterns_from" also now expand "~". + * #1023: Fix an error in the Btrfs hook when attempting to snapshot a read-only subvolume. Now, + read-only subvolumes are ignored since Btrfs can't actually snapshot them. + +1.9.13 + * #975: Add a "compression" option to the PostgreSQL database hook. + * #1001: Fix a ZFS error during snapshot cleanup. + * #1003: In the Zabbix monitoring hook, support Zabbix 7.2's authentication changes. + * #1009: Send database passwords to MariaDB and MySQL via anonymous pipe, which is more secure than + using an environment variable. + * #1013: Send database passwords to MongoDB via anonymous pipe, which is more secure than using + "--password" on the command-line! + * #1015: When ctrl-C is pressed, more strongly encourage Borg to actually exit. + * Add a "verify_tls" option to the Uptime Kuma monitoring hook for disabling TLS verification. + * Add "tls" options to the MariaDB and MySQL database hooks to enable or disable TLS encryption + between client and server. + +1.9.12 + * #1005: Fix the credential hooks to avoid using Python 3.12+ string features. Now borgmatic will + work with Python 3.9, 3.10, and 3.11 again. + +1.9.11 + * #795: Add credential loading from file, KeePassXC, and Docker/Podman secrets. See the + documentation for more information: + https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/ + * #996: Fix the "create" action to omit the repository label prefix from Borg's output when + databases are enabled. + * #998: Send the "encryption_passphrase" option to Borg via an anonymous pipe, which is more secure + than using an environment variable. + * #999: Fix a runtime directory error from a conflict between "extra_borg_options" and special file + detection. + * #1001: For the ZFS, Btrfs, and LVM hooks, only make snapshots for root patterns that come from + a borgmatic configuration option (e.g. "source_directories")—not from other hooks within + borgmatic. + * #1001: Fix a ZFS/LVM error due to colliding snapshot mount points for nested datasets or logical + volumes. + * #1001: Don't try to snapshot ZFS datasets that have the "canmount=off" property. + * Fix another error in the Btrfs hook when a subvolume mounted at "/" is configured in borgmatic's + source directories. + +1.9.10 + * #966: Add a "{credential ...}" syntax for loading systemd credentials into borgmatic + configuration files. See the documentation for more information: + https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/ * #987: Fix a "list" action error when the "encryption_passcommand" option is set. * #987: When both "encryption_passcommand" and "encryption_passphrase" are configured, prefer "encryption_passphrase" even if it's an empty value. @@ -7,6 +98,8 @@ refused to run checks in this situation. * #989: Fix the log message code to avoid using Python 3.10+ logging features. Now borgmatic will work with Python 3.9 again. + * Capture and delay any log records produced before logging is fully configured, so early log + records don't get lost. * Add support for Python 3.13. 1.9.9 diff --git a/README.md b/README.md index b57f68d4..925b2b33 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,8 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/). ## Integrations +### Data + PostgreSQL MySQL MariaDB @@ -65,6 +67,11 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/). Btrfs LVM rclone +BorgBase + + +### Monitoring + Healthchecks Uptime Kuma Cronitor @@ -76,7 +83,14 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/). Apprise Zabbix Sentry -BorgBase + + +### Credentials + +Sentry +Docker +Podman +Podman ## Getting started diff --git a/borgmatic/actions/change_passphrase.py b/borgmatic/actions/change_passphrase.py index 7c4a3974..6d0ff31c 100644 --- a/borgmatic/actions/change_passphrase.py +++ b/borgmatic/actions/change_passphrase.py @@ -16,7 +16,7 @@ def run_change_passphrase( remote_path, ): ''' - Run the "key change-passprhase" action for the given repository. + Run the "key change-passphrase" action for the given repository. ''' if ( change_passphrase_arguments.repository is None diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index d3e347e3..13023ff9 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -170,7 +170,7 @@ def filter_checks_on_frequency( if calendar.day_name[datetime_now().weekday()] not in days: logger.info( - f"Skipping {check} check due to day of the week; check only runs on {'/'.join(days)} (use --force to check anyway)" + f"Skipping {check} check due to day of the week; check only runs on {'/'.join(day.title() for day in days)} (use --force to check anyway)" ) filtered_checks.remove(check) continue @@ -372,7 +372,7 @@ def collect_spot_check_source_paths( borgmatic.borg.create.make_base_create_command( dry_run=True, repository_path=repository['path'], - config=config, + config=dict(config, list_details=True), patterns=borgmatic.actions.create.process_patterns( borgmatic.actions.create.collect_patterns(config), working_directory, @@ -382,7 +382,6 @@ def collect_spot_check_source_paths( borgmatic_runtime_directory=borgmatic_runtime_directory, local_path=local_path, remote_path=remote_path, - list_files=True, stream_processes=stream_processes, ) ) @@ -391,7 +390,7 @@ def collect_spot_check_source_paths( paths_output = borgmatic.execute.execute_command_and_capture_output( create_flags + create_positional_arguments, capture_stderr=True, - extra_environment=borgmatic.borg.environment.make_environment(config), + environment=borgmatic.borg.environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), @@ -483,10 +482,12 @@ def compare_spot_check_hashes( ) source_sample_paths = tuple(random.sample(source_paths, sample_count)) working_directory = borgmatic.config.paths.get_working_directory(config) - existing_source_sample_paths = { + hashable_source_sample_path = { source_path for source_path in source_sample_paths - if os.path.exists(os.path.join(working_directory or '', source_path)) + for full_source_path in (os.path.join(working_directory or '', source_path),) + if os.path.exists(full_source_path) + if not os.path.islink(full_source_path) } logger.debug( f'Sampling {sample_count} source paths (~{spot_check_config["data_sample_percentage"]}%) for spot check' @@ -509,7 +510,7 @@ def compare_spot_check_hashes( hash_output = borgmatic.execute.execute_command_and_capture_output( (spot_check_config.get('xxh64sum_command', 'xxh64sum'),) + tuple( - path for path in source_sample_paths_subset if path in existing_source_sample_paths + path for path in source_sample_paths_subset if path in hashable_source_sample_path ), working_directory=working_directory, ) @@ -517,11 +518,13 @@ def compare_spot_check_hashes( source_hashes.update( **dict( (reversed(line.split(' ', 1)) for line in hash_output.splitlines()), - # Represent non-existent files as having empty hashes so the comparison below still works. + # Represent non-existent files as having empty hashes so the comparison below still + # works. Same thing for filesystem links, since Borg produces empty archive hashes + # for them. **{ path: '' for path in source_sample_paths_subset - if path not in existing_source_sample_paths + if path not in hashable_source_sample_path }, ) ) @@ -682,7 +685,6 @@ def run_check( config_filename, repository, config, - hook_context, local_borg_version, check_arguments, global_arguments, @@ -699,15 +701,6 @@ def run_check( ): return - borgmatic.hooks.command.execute_hook( - config.get('before_check'), - config.get('umask'), - config_filename, - 'pre-check', - global_arguments.dry_run, - **hook_context, - ) - logger.info('Running consistency checks') repository_id = borgmatic.borg.check.get_repository_id( @@ -772,12 +765,3 @@ def run_check( borgmatic_runtime_directory, ) write_check_time(make_check_time_path(config, repository_id, 'spot')) - - borgmatic.hooks.command.execute_hook( - config.get('after_check'), - config.get('umask'), - config_filename, - 'post-check', - global_arguments.dry_run, - **hook_context, - ) diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index b0b070bf..551c680d 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -12,7 +12,6 @@ def run_compact( config_filename, repository, config, - hook_context, local_borg_version, compact_arguments, global_arguments, @@ -28,14 +27,6 @@ def run_compact( ): return - borgmatic.hooks.command.execute_hook( - config.get('before_compact'), - config.get('umask'), - config_filename, - 'pre-compact', - global_arguments.dry_run, - **hook_context, - ) if borgmatic.borg.feature.available(borgmatic.borg.feature.Feature.COMPACT, local_borg_version): logger.info(f'Compacting segments{dry_run_label}') borgmatic.borg.compact.compact_segments( @@ -46,18 +37,7 @@ def run_compact( global_arguments, local_path=local_path, remote_path=remote_path, - progress=compact_arguments.progress, cleanup_commits=compact_arguments.cleanup_commits, - threshold=compact_arguments.threshold, ) else: # pragma: nocover logger.info('Skipping compact (only available/needed in Borg 1.2+)') - - borgmatic.hooks.command.execute_hook( - config.get('after_compact'), - config.get('umask'), - config_filename, - 'post-compact', - global_arguments.dry_run, - **hook_context, - ) diff --git a/borgmatic/actions/config/bootstrap.py b/borgmatic/actions/config/bootstrap.py index 4a72b2cd..520bcb1c 100644 --- a/borgmatic/actions/config/bootstrap.py +++ b/borgmatic/actions/config/bootstrap.py @@ -119,7 +119,9 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version): bootstrap_arguments.repository, archive_name, [config_path.lstrip(os.path.sep) for config_path in manifest_config_paths], - config, + # Only add progress here and not the extract_archive() call above, because progress + # conflicts with extract_to_stdout. + dict(config, progress=bootstrap_arguments.progress or False), local_borg_version, global_arguments, local_path=bootstrap_arguments.local_path, @@ -127,5 +129,4 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version): extract_to_stdout=False, destination_path=bootstrap_arguments.destination, strip_components=bootstrap_arguments.strip_components, - progress=bootstrap_arguments.progress, ) diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index ab7815e4..a92af261 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -36,6 +36,7 @@ def parse_pattern(pattern_line, default_style=borgmatic.borg.pattern.Pattern_sty path, borgmatic.borg.pattern.Pattern_type(pattern_type), borgmatic.borg.pattern.Pattern_style(pattern_style), + source=borgmatic.borg.pattern.Pattern_source.CONFIG, ) @@ -51,7 +52,9 @@ def collect_patterns(config): try: return ( tuple( - borgmatic.borg.pattern.Pattern(source_directory) + borgmatic.borg.pattern.Pattern( + source_directory, source=borgmatic.borg.pattern.Pattern_source.CONFIG + ) for source_directory in config.get('source_directories', ()) ) + tuple( @@ -127,8 +130,11 @@ def expand_directory(directory, working_directory): def expand_patterns(patterns, working_directory=None, skip_paths=None): ''' Given a sequence of borgmatic.borg.pattern.Pattern instances and an optional working directory, - expand tildes and globs in each root pattern. Return all the resulting patterns (not just the - root patterns) as a tuple. + expand tildes and globs in each root pattern and expand just tildes in each non-root pattern. + The idea is that non-root patterns may be regular expressions or other pattern styles containing + "*" that borgmatic should not expand as a shell glob. + + Return all the resulting patterns as a tuple. If a set of paths are given to skip, then don't expand any patterns matching them. ''' @@ -144,12 +150,21 @@ def expand_patterns(patterns, working_directory=None, skip_paths=None): pattern.type, pattern.style, pattern.device, + pattern.source, ) for expanded_path in expand_directory(pattern.path, working_directory) ) if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT and pattern.path not in (skip_paths or ()) - else (pattern,) + else ( + borgmatic.borg.pattern.Pattern( + os.path.expanduser(pattern.path), + pattern.type, + pattern.style, + pattern.device, + pattern.source, + ), + ) ) for pattern in patterns ) @@ -178,6 +193,7 @@ def device_map_patterns(patterns, working_directory=None): and os.path.exists(full_path) else None ), + source=pattern.source, ) for pattern in patterns for full_path in (os.path.join(working_directory or '', pattern.path),) @@ -256,7 +272,6 @@ def run_create( repository, config, config_paths, - hook_context, local_borg_version, create_arguments, global_arguments, @@ -274,14 +289,15 @@ def run_create( ): return - borgmatic.hooks.command.execute_hook( - config.get('before_backup'), - config.get('umask'), - config_filename, - 'pre-backup', - global_arguments.dry_run, - **hook_context, - ) + if config.get('list_details') and config.get('progress'): + raise ValueError( + 'With the create action, only one of --list/--files/list_details and --progress/progress can be used.' + ) + + if config.get('list_details') and create_arguments.json: + raise ValueError( + 'With the create action, only one of --list/--files/list_details and --json can be used.' + ) logger.info(f'Creating archive{dry_run_label}') working_directory = borgmatic.config.paths.get_working_directory(config) @@ -321,10 +337,7 @@ def run_create( borgmatic_runtime_directory, local_path=local_path, remote_path=remote_path, - progress=create_arguments.progress, - stats=create_arguments.stats, json=create_arguments.json, - list_files=create_arguments.list_files, stream_processes=stream_processes, ) @@ -338,12 +351,3 @@ def run_create( borgmatic_runtime_directory, global_arguments.dry_run, ) - - borgmatic.hooks.command.execute_hook( - config.get('after_backup'), - config.get('umask'), - config_filename, - 'post-backup', - global_arguments.dry_run, - **hook_context, - ) diff --git a/borgmatic/actions/export_tar.py b/borgmatic/actions/export_tar.py index d5c6bacb..f04b06ff 100644 --- a/borgmatic/actions/export_tar.py +++ b/borgmatic/actions/export_tar.py @@ -43,6 +43,5 @@ def run_export_tar( local_path=local_path, remote_path=remote_path, tar_filter=export_tar_arguments.tar_filter, - list_files=export_tar_arguments.list_files, strip_components=export_tar_arguments.strip_components, ) diff --git a/borgmatic/actions/extract.py b/borgmatic/actions/extract.py index 0216f820..feaaac81 100644 --- a/borgmatic/actions/extract.py +++ b/borgmatic/actions/extract.py @@ -12,7 +12,6 @@ def run_extract( config_filename, repository, config, - hook_context, local_borg_version, extract_arguments, global_arguments, @@ -22,14 +21,6 @@ def run_extract( ''' Run the "extract" action for the given repository. ''' - borgmatic.hooks.command.execute_hook( - config.get('before_extract'), - config.get('umask'), - config_filename, - 'pre-extract', - global_arguments.dry_run, - **hook_context, - ) if extract_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, extract_arguments.repository ): @@ -54,13 +45,4 @@ def run_extract( remote_path=remote_path, destination_path=extract_arguments.destination, strip_components=extract_arguments.strip_components, - progress=extract_arguments.progress, ) - borgmatic.hooks.command.execute_hook( - config.get('after_extract'), - config.get('umask'), - config_filename, - 'post-extract', - global_arguments.dry_run, - **hook_context, - ) diff --git a/borgmatic/actions/import_key.py b/borgmatic/actions/import_key.py new file mode 100644 index 00000000..42813efa --- /dev/null +++ b/borgmatic/actions/import_key.py @@ -0,0 +1,33 @@ +import logging + +import borgmatic.borg.import_key +import borgmatic.config.validate + +logger = logging.getLogger(__name__) + + +def run_import_key( + repository, + config, + local_borg_version, + import_arguments, + global_arguments, + local_path, + remote_path, +): + ''' + Run the "key import" action for the given repository. + ''' + if import_arguments.repository is None or borgmatic.config.validate.repositories_match( + repository, import_arguments.repository + ): + logger.info('Importing repository key') + borgmatic.borg.import_key.import_key( + repository['path'], + config, + local_borg_version, + import_arguments, + global_arguments, + local_path=local_path, + remote_path=remote_path, + ) diff --git a/borgmatic/actions/prune.py b/borgmatic/actions/prune.py index 83dfe39f..e7caa158 100644 --- a/borgmatic/actions/prune.py +++ b/borgmatic/actions/prune.py @@ -11,7 +11,6 @@ def run_prune( config_filename, repository, config, - hook_context, local_borg_version, prune_arguments, global_arguments, @@ -27,14 +26,6 @@ def run_prune( ): return - borgmatic.hooks.command.execute_hook( - config.get('before_prune'), - config.get('umask'), - config_filename, - 'pre-prune', - global_arguments.dry_run, - **hook_context, - ) logger.info(f'Pruning archives{dry_run_label}') borgmatic.borg.prune.prune_archives( global_arguments.dry_run, @@ -46,11 +37,3 @@ def run_prune( local_path=local_path, remote_path=remote_path, ) - borgmatic.hooks.command.execute_hook( - config.get('after_prune'), - config.get('umask'), - config_filename, - 'post-prune', - global_arguments.dry_run, - **hook_context, - ) diff --git a/borgmatic/actions/recreate.py b/borgmatic/actions/recreate.py new file mode 100644 index 00000000..3cef7762 --- /dev/null +++ b/borgmatic/actions/recreate.py @@ -0,0 +1,53 @@ +import logging + +import borgmatic.borg.recreate +import borgmatic.config.validate +from borgmatic.actions.create import collect_patterns, process_patterns + +logger = logging.getLogger(__name__) + + +def run_recreate( + repository, + config, + local_borg_version, + recreate_arguments, + global_arguments, + local_path, + remote_path, +): + ''' + Run the "recreate" action for the given repository. + ''' + if recreate_arguments.repository is None or borgmatic.config.validate.repositories_match( + repository, recreate_arguments.repository + ): + if recreate_arguments.archive: + logger.answer(f'Recreating archive {recreate_arguments.archive}') + else: + logger.answer('Recreating repository') + + # Collect and process patterns. + processed_patterns = process_patterns( + collect_patterns(config), borgmatic.config.paths.get_working_directory(config) + ) + + borgmatic.borg.recreate.recreate_archive( + repository['path'], + borgmatic.borg.repo_list.resolve_archive_name( + repository['path'], + recreate_arguments.archive, + config, + local_borg_version, + global_arguments, + local_path, + remote_path, + ), + config, + local_borg_version, + recreate_arguments, + global_arguments, + local_path=local_path, + remote_path=remote_path, + patterns=processed_patterns, + ) diff --git a/borgmatic/actions/repo_create.py b/borgmatic/actions/repo_create.py index e6252e4e..38d35922 100644 --- a/borgmatic/actions/repo_create.py +++ b/borgmatic/actions/repo_create.py @@ -24,18 +24,38 @@ def run_repo_create( return logger.info('Creating repository') + + encryption_mode = repo_create_arguments.encryption_mode or repository.get('encryption') + + if not encryption_mode: + raise ValueError( + 'With the repo-create action, either the --encryption flag or the repository encryption option is required.' + ) + borgmatic.borg.repo_create.create_repository( global_arguments.dry_run, repository['path'], config, local_borg_version, global_arguments, - repo_create_arguments.encryption_mode, + encryption_mode, repo_create_arguments.source_repository, repo_create_arguments.copy_crypt_key, - repo_create_arguments.append_only, - repo_create_arguments.storage_quota, - repo_create_arguments.make_parent_dirs, + ( + repository.get('append_only') + if repo_create_arguments.append_only is None + else repo_create_arguments.append_only + ), + ( + repository.get('storage_quota') + if repo_create_arguments.storage_quota is None + else repo_create_arguments.storage_quota + ), + ( + repository.get('make_parent_directories') + if repo_create_arguments.make_parent_directories is None + else repo_create_arguments.make_parent_directories + ), local_path=local_path, remote_path=remote_path, ) diff --git a/borgmatic/actions/transfer.py b/borgmatic/actions/transfer.py index 8a27d16b..c9601515 100644 --- a/borgmatic/actions/transfer.py +++ b/borgmatic/actions/transfer.py @@ -17,7 +17,13 @@ def run_transfer( ''' Run the "transfer" action for the given repository. ''' + if transfer_arguments.archive and config.get('match_archives'): + raise ValueError( + 'With the transfer action, only one of --archive and --match-archives/match_archives can be used.' + ) + logger.info('Transferring archives to repository') + borgmatic.borg.transfer.transfer_archives( global_arguments.dry_run, repository['path'], diff --git a/borgmatic/borg/borg.py b/borgmatic/borg/borg.py index eb52894a..a2afd430 100644 --- a/borgmatic/borg/borg.py +++ b/borgmatic/borg/borg.py @@ -61,7 +61,7 @@ def run_arbitrary_borg( tuple(shlex.quote(part) for part in full_command), output_file=DO_NOT_CAPTURE, shell=True, - extra_environment=dict( + environment=dict( (environment.make_environment(config) or {}), **{ 'BORG_REPO': repository_path, diff --git a/borgmatic/borg/break_lock.py b/borgmatic/borg/break_lock.py index 478221f0..8c5533ce 100644 --- a/borgmatic/borg/break_lock.py +++ b/borgmatic/borg/break_lock.py @@ -36,7 +36,7 @@ def break_lock( execute_command( full_command, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), diff --git a/borgmatic/borg/change_passphrase.py b/borgmatic/borg/change_passphrase.py index 79f9781d..f527e204 100644 --- a/borgmatic/borg/change_passphrase.py +++ b/borgmatic/borg/change_passphrase.py @@ -56,7 +56,7 @@ def change_passphrase( full_command, output_file=borgmatic.execute.DO_NOT_CAPTURE, output_log_level=logging.ANSWER, - extra_environment=environment.make_environment(config_without_passphrase), + environment=environment.make_environment(config_without_passphrase), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), diff --git a/borgmatic/borg/check.py b/borgmatic/borg/check.py index 4350b8e4..935c52c8 100644 --- a/borgmatic/borg/check.py +++ b/borgmatic/borg/check.py @@ -32,7 +32,7 @@ def make_archive_filter_flags(local_borg_version, config, checks, check_argument if prefix else ( flags.make_match_archives_flags( - check_arguments.match_archives or config.get('match_archives'), + config.get('match_archives'), config.get('archive_name_format'), local_borg_version, ) @@ -170,7 +170,7 @@ def check_archives( + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + verbosity_flags - + (('--progress',) if check_arguments.progress else ()) + + (('--progress',) if config.get('progress') else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + flags.make_repository_flags(repository_path, local_borg_version) ) @@ -180,9 +180,9 @@ def check_archives( # The Borg repair option triggers an interactive prompt, which won't work when output is # captured. And progress messes with the terminal directly. output_file=( - DO_NOT_CAPTURE if check_arguments.repair or check_arguments.progress else None + DO_NOT_CAPTURE if check_arguments.repair or config.get('progress') else None ), - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, borg_exit_codes=borg_exit_codes, diff --git a/borgmatic/borg/compact.py b/borgmatic/borg/compact.py index eb16c6fa..b443e8c0 100644 --- a/borgmatic/borg/compact.py +++ b/borgmatic/borg/compact.py @@ -15,9 +15,7 @@ def compact_segments( global_arguments, local_path='borg', remote_path=None, - progress=False, cleanup_commits=False, - threshold=None, ): ''' Given dry-run flag, a local or remote repository path, a configuration dict, and the local Borg @@ -26,6 +24,7 @@ def compact_segments( umask = config.get('umask', None) lock_wait = config.get('lock_wait', None) extra_borg_options = config.get('extra_borg_options', {}).get('compact', '') + threshold = config.get('compact_threshold') full_command = ( (local_path, 'compact') @@ -33,7 +32,7 @@ def compact_segments( + (('--umask', str(umask)) if umask else ()) + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) - + (('--progress',) if progress else ()) + + (('--progress',) if config.get('progress') else ()) + (('--cleanup-commits',) if cleanup_commits else ()) + (('--threshold', str(threshold)) if threshold else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) @@ -49,7 +48,7 @@ def compact_segments( execute_command( full_command, output_log_level=logging.INFO, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 924c50e7..de5115dc 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -132,41 +132,53 @@ def collect_special_file_paths( used. Skip looking for special files in the given borgmatic runtime directory, as borgmatic creates - its own special files there for database dumps. And if the borgmatic runtime directory is - configured to be excluded from the files Borg backs up, error, because this means Borg won't be - able to consume any database dumps and therefore borgmatic will hang. + its own special files there for database dumps and we don't want those omitted. + + Additionally, if the borgmatic runtime directory is not contained somewhere in the files Borg + plans to backup, that means the user must have excluded the runtime directory (e.g. via + "exclude_patterns" or similar). Therefore, raise, because this means Borg won't be able to + consume any database dumps and therefore borgmatic will hang when it tries to do so. ''' # Omit "--exclude-nodump" from the Borg dry run command, because that flag causes Borg to open - # files including any named pipe we've created. + # files including any named pipe we've created. And omit "--filter" because that can break the + # paths output parsing below such that path lines no longer start with th expected "- ". paths_output = execute_command_and_capture_output( - tuple(argument for argument in create_command if argument != '--exclude-nodump') + flags.omit_flag_and_value(flags.omit_flag(create_command, '--exclude-nodump'), '--filter') + ('--dry-run', '--list'), capture_stderr=True, working_directory=working_directory, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), ) + # These are all the individual files that Borg is planning to backup as determined by the Borg + # create dry run above. paths = tuple( path_line.split(' ', 1)[1] for path_line in paths_output.split('\n') if path_line and path_line.startswith('- ') or path_line.startswith('+ ') ) - skip_paths = {} + + # These are the subset of those files that contain the borgmatic runtime directory. + paths_containing_runtime_directory = {} if os.path.exists(borgmatic_runtime_directory): - skip_paths = { + paths_containing_runtime_directory = { path for path in paths if any_parent_directories(path, (borgmatic_runtime_directory,)) } - if not skip_paths and not dry_run: + # If no paths to backup contain the runtime directory, it must've been excluded. + if not paths_containing_runtime_directory and not dry_run: raise ValueError( f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes or patterns with excludes. Please ensure the runtime directory is not excluded.' ) return tuple( - path for path in paths if special_file(path, working_directory) if path not in skip_paths + path + for path in paths + if special_file(path, working_directory) + if path not in paths_containing_runtime_directory ) @@ -184,7 +196,7 @@ def check_all_root_patterns_exist(patterns): if missing_paths: raise ValueError( - f"Source directories / root pattern paths do not exist: {', '.join(missing_paths)}" + f"Source directories or root pattern paths do not exist: {', '.join(missing_paths)}" ) @@ -201,9 +213,7 @@ def make_base_create_command( borgmatic_runtime_directory, local_path='borg', remote_path=None, - progress=False, json=False, - list_files=False, stream_processes=None, ): ''' @@ -281,7 +291,7 @@ def make_base_create_command( + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + ( ('--list', '--filter', list_filter_flags) - if list_files and not json and not progress + if config.get('list_details') and not json and not config.get('progress') else () ) + (('--dry-run',) if dry_run else ()) @@ -325,6 +335,7 @@ def make_base_create_command( special_file_path, borgmatic.borg.pattern.Pattern_type.NO_RECURSE, borgmatic.borg.pattern.Pattern_style.FNMATCH, + source=borgmatic.borg.pattern.Pattern_source.INTERNAL, ) for special_file_path in special_file_paths ), @@ -348,10 +359,7 @@ def create_archive( borgmatic_runtime_directory, local_path='borg', remote_path=None, - progress=False, - stats=False, json=False, - list_files=False, stream_processes=None, ): ''' @@ -376,28 +384,26 @@ def create_archive( borgmatic_runtime_directory, local_path, remote_path, - progress, json, - list_files, stream_processes, ) if json: output_log_level = None - elif list_files or (stats and not dry_run): + elif config.get('list_details') or (config.get('statistics') and not dry_run): output_log_level = logging.ANSWER else: output_log_level = logging.INFO # The progress output isn't compatible with captured and logged output, as progress messes with # the terminal directly. - output_file = DO_NOT_CAPTURE if progress else None + output_file = DO_NOT_CAPTURE if config.get('progress') else None create_flags += ( (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ()) - + (('--stats',) if stats and not json and not dry_run else ()) + + (('--stats',) if config.get('statistics') and not json and not dry_run else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else ()) - + (('--progress',) if progress else ()) + + (('--progress',) if config.get('progress') else ()) + (('--json',) if json else ()) ) borg_exit_codes = config.get('borg_exit_codes') @@ -409,7 +415,7 @@ def create_archive( output_log_level, output_file, working_directory=working_directory, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), borg_local_path=local_path, borg_exit_codes=borg_exit_codes, ) @@ -417,7 +423,7 @@ def create_archive( return execute_command_and_capture_output( create_flags + create_positional_arguments, working_directory=working_directory, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), borg_local_path=local_path, borg_exit_codes=borg_exit_codes, ) @@ -427,7 +433,7 @@ def create_archive( output_log_level, output_file, working_directory=working_directory, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), borg_local_path=local_path, borg_exit_codes=borg_exit_codes, ) diff --git a/borgmatic/borg/delete.py b/borgmatic/borg/delete.py index d53a1561..63ee0ef8 100644 --- a/borgmatic/borg/delete.py +++ b/borgmatic/borg/delete.py @@ -34,7 +34,7 @@ def make_delete_command( + borgmatic.borg.flags.make_flags('umask', config.get('umask')) + borgmatic.borg.flags.make_flags('log-json', global_arguments.log_json) + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait')) - + borgmatic.borg.flags.make_flags('list', delete_arguments.list_archives) + + borgmatic.borg.flags.make_flags('list', config.get('list_details')) + ( (('--force',) + (('--force',) if delete_arguments.force >= 2 else ())) if delete_arguments.force @@ -48,9 +48,17 @@ def make_delete_command( local_borg_version=local_borg_version, default_archive_name_format='*', ) + + (('--stats',) if config.get('statistics') else ()) + borgmatic.borg.flags.make_flags_from_arguments( delete_arguments, - excludes=('list_archives', 'force', 'match_archives', 'archive', 'repository'), + excludes=( + 'list_details', + 'statistics', + 'force', + 'match_archives', + 'archive', + 'repository', + ), ) + borgmatic.borg.flags.make_repository_flags(repository['path'], local_borg_version) ) @@ -98,7 +106,7 @@ def delete_archives( repo_delete_arguments = argparse.Namespace( repository=repository['path'], - list_archives=delete_arguments.list_archives, + list_details=delete_arguments.list_details, force=delete_arguments.force, cache_only=delete_arguments.cache_only, keep_security_info=delete_arguments.keep_security_info, @@ -128,7 +136,7 @@ def delete_archives( borgmatic.execute.execute_command( command, output_log_level=logging.ANSWER, - extra_environment=borgmatic.borg.environment.make_environment(config), + environment=borgmatic.borg.environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), diff --git a/borgmatic/borg/environment.py b/borgmatic/borg/environment.py index 839d0887..344478c2 100644 --- a/borgmatic/borg/environment.py +++ b/borgmatic/borg/environment.py @@ -1,6 +1,7 @@ import os import borgmatic.borg.passcommand +import borgmatic.hooks.credential.parse OPTION_TO_ENVIRONMENT_VARIABLE = { 'borg_base_directory': 'BORG_BASE_DIR', @@ -9,7 +10,6 @@ OPTION_TO_ENVIRONMENT_VARIABLE = { 'borg_files_cache_ttl': 'BORG_FILES_CACHE_TTL', 'borg_security_directory': 'BORG_SECURITY_DIR', 'borg_keys_directory': 'BORG_KEYS_DIR', - 'encryption_passphrase': 'BORG_PASSPHRASE', 'ssh_command': 'BORG_RSH', 'temporary_directory': 'TMPDIR', } @@ -26,29 +26,55 @@ DEFAULT_BOOL_OPTION_TO_UPPERCASE_ENVIRONMENT_VARIABLE = { def make_environment(config): ''' - Given a borgmatic configuration dict, return its options converted to a Borg environment - variable dict. + Given a borgmatic configuration dict, convert it to a Borg environment variable dict, merge it + with a copy of the current environment variables, and return the result. Do not reuse this environment across multiple Borg invocations, because it can include references to resources like anonymous pipes for passphrases—which can only be consumed once. + + Here's how native Borg precedence works for a few of the environment variables: + + 1. BORG_PASSPHRASE, if set, is used first. + 2. BORG_PASSCOMMAND is used only if BORG_PASSPHRASE isn't set. + 3. BORG_PASSPHRASE_FD is used only if neither of the above are set. + + In borgmatic, we want to simulate this precedence order, but there are some additional + complications. First, values can come from either configuration or from environment variables + set outside borgmatic; configured options should take precedence. Second, when borgmatic gets a + passphrase—directly from configuration or indirectly via a credential hook or a passcommand—we + want to pass that passphrase to Borg via an anonymous pipe (+ BORG_PASSPHRASE_FD), since that's + more secure than using an environment variable (BORG_PASSPHRASE). ''' - environment = {} + environment = dict(os.environ) for option_name, environment_variable_name in OPTION_TO_ENVIRONMENT_VARIABLE.items(): value = config.get(option_name) - if value: + if value is not None: environment[environment_variable_name] = str(value) - passphrase = borgmatic.borg.passcommand.get_passphrase_from_passcommand(config) + if 'encryption_passphrase' in config: + environment.pop('BORG_PASSPHRASE', None) + environment.pop('BORG_PASSCOMMAND', None) - # If the passcommand produced a passphrase, send it to Borg via an anonymous pipe. - if passphrase: + if 'encryption_passcommand' in config: + environment.pop('BORG_PASSCOMMAND', None) + + passphrase = borgmatic.hooks.credential.parse.resolve_credential( + config.get('encryption_passphrase'), config + ) + + if passphrase is None: + passphrase = borgmatic.borg.passcommand.get_passphrase_from_passcommand(config) + + # If there's a passphrase (from configuration, from a configured credential, or from a + # configured passcommand), send it to Borg via an anonymous pipe. + if passphrase is not None: read_file_descriptor, write_file_descriptor = os.pipe() os.write(write_file_descriptor, passphrase.encode('utf-8')) os.close(write_file_descriptor) - # This, plus subprocess.Popen(..., close_fds=False) in execute.py, is necessary for the Borg + # This plus subprocess.Popen(..., close_fds=False) in execute.py is necessary for the Borg # child process to inherit the file descriptor. os.set_inheritable(read_file_descriptor, True) environment['BORG_PASSPHRASE_FD'] = str(read_file_descriptor) diff --git a/borgmatic/borg/export_key.py b/borgmatic/borg/export_key.py index 768b52cc..0d050b16 100644 --- a/borgmatic/borg/export_key.py +++ b/borgmatic/borg/export_key.py @@ -67,7 +67,7 @@ def export_key( full_command, output_file=output_file, output_log_level=logging.ANSWER, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), diff --git a/borgmatic/borg/export_tar.py b/borgmatic/borg/export_tar.py index 728755d5..224c07b1 100644 --- a/borgmatic/borg/export_tar.py +++ b/borgmatic/borg/export_tar.py @@ -20,7 +20,6 @@ def export_tar_archive( local_path='borg', remote_path=None, tar_filter=None, - list_files=False, strip_components=None, ): ''' @@ -43,7 +42,7 @@ def export_tar_archive( + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) - + (('--list',) if list_files else ()) + + (('--list',) if config.get('list_details') else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (('--dry-run',) if dry_run else ()) + (('--tar-filter', tar_filter) if tar_filter else ()) @@ -57,7 +56,7 @@ def export_tar_archive( + (tuple(paths) if paths else ()) ) - if list_files: + if config.get('list_details'): output_log_level = logging.ANSWER else: output_log_level = logging.INFO @@ -70,7 +69,7 @@ def export_tar_archive( full_command, output_file=DO_NOT_CAPTURE if destination_path == '-' else None, output_log_level=output_log_level, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), diff --git a/borgmatic/borg/extract.py b/borgmatic/borg/extract.py index d599b5b6..aa354cbc 100644 --- a/borgmatic/borg/extract.py +++ b/borgmatic/borg/extract.py @@ -58,7 +58,7 @@ def extract_last_archive_dry_run( execute_command( full_extract_command, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), @@ -77,7 +77,6 @@ def extract_archive( remote_path=None, destination_path=None, strip_components=None, - progress=False, extract_to_stdout=False, ): ''' @@ -92,8 +91,8 @@ def extract_archive( umask = config.get('umask', None) lock_wait = config.get('lock_wait', None) - if progress and extract_to_stdout: - raise ValueError('progress and extract_to_stdout cannot both be set') + if config.get('progress') and extract_to_stdout: + raise ValueError('progress and extract to stdout cannot both be set') if feature.available(feature.Feature.NUMERIC_IDS, local_borg_version): numeric_ids_flags = ('--numeric-ids',) if config.get('numeric_ids') else () @@ -128,15 +127,13 @@ def extract_archive( + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (('--dry-run',) if dry_run else ()) + (('--strip-components', str(strip_components)) if strip_components else ()) - + (('--progress',) if progress else ()) + + (('--progress',) if config.get('progress') else ()) + (('--stdout',) if extract_to_stdout else ()) + flags.make_repository_archive_flags( # Make the repository path absolute so the destination directory used below via changing # the working directory doesn't prevent Borg from finding the repo. But also apply the # user's configured working directory (if any) to the repo path. - borgmatic.config.validate.normalize_repository_path( - os.path.join(working_directory or '', repository) - ), + borgmatic.config.validate.normalize_repository_path(repository, working_directory), archive, local_borg_version, ) @@ -150,11 +147,11 @@ def extract_archive( # The progress output isn't compatible with captured and logged output, as progress messes with # the terminal directly. - if progress: + if config.get('progress'): return execute_command( full_command, output_file=DO_NOT_CAPTURE, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=full_destination_path, borg_local_path=local_path, borg_exit_codes=borg_exit_codes, @@ -166,7 +163,7 @@ def extract_archive( full_command, output_file=subprocess.PIPE, run_to_completion=False, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=full_destination_path, borg_local_path=local_path, borg_exit_codes=borg_exit_codes, @@ -176,7 +173,7 @@ def extract_archive( # if the restore paths don't exist in the archive. execute_command( full_command, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=full_destination_path, borg_local_path=local_path, borg_exit_codes=borg_exit_codes, diff --git a/borgmatic/borg/feature.py b/borgmatic/borg/feature.py index a6462e13..c45899c0 100644 --- a/borgmatic/borg/feature.py +++ b/borgmatic/borg/feature.py @@ -17,6 +17,7 @@ class Feature(Enum): MATCH_ARCHIVES = 11 EXCLUDED_FILES_MINUS = 12 ARCHIVE_SERIES = 13 + NO_PRUNE_STATS = 14 FEATURE_TO_MINIMUM_BORG_VERSION = { @@ -33,6 +34,7 @@ FEATURE_TO_MINIMUM_BORG_VERSION = { Feature.MATCH_ARCHIVES: parse('2.0.0b3'), # borg --match-archives Feature.EXCLUDED_FILES_MINUS: parse('2.0.0b5'), # --list --filter uses "-" for excludes Feature.ARCHIVE_SERIES: parse('2.0.0b11'), # identically named archives form a series + Feature.NO_PRUNE_STATS: parse('2.0.0b10'), # prune --stats is not available } diff --git a/borgmatic/borg/flags.py b/borgmatic/borg/flags.py index 97bda27d..a88400e1 100644 --- a/borgmatic/borg/flags.py +++ b/borgmatic/borg/flags.py @@ -156,3 +156,44 @@ def warn_for_aggressive_archive_flags(json_command, json_output): logger.debug(f'Cannot parse JSON output from archive command: {error}') except (TypeError, KeyError): logger.debug('Cannot parse JSON output from archive command: No "archives" key found') + + +def omit_flag(arguments, flag): + ''' + Given a sequence of Borg command-line arguments, return them with the given (valueless) flag + omitted. For instance, if the flag is "--flag" and arguments is: + + ('borg', 'create', '--flag', '--other-flag') + + ... then return: + + ('borg', 'create', '--other-flag') + ''' + return tuple(argument for argument in arguments if argument != flag) + + +def omit_flag_and_value(arguments, flag): + ''' + Given a sequence of Borg command-line arguments, return them with the given flag and its + corresponding value omitted. For instance, if the flag is "--flag" and arguments is: + + ('borg', 'create', '--flag', 'value', '--other-flag') + + ... or: + + ('borg', 'create', '--flag=value', '--other-flag') + + ... then return: + + ('borg', 'create', '--other-flag') + ''' + # This works by zipping together a list of overlapping pairwise arguments. E.g., ('one', 'two', + # 'three', 'four') becomes ((None, 'one'), ('one, 'two'), ('two', 'three'), ('three', 'four')). + # This makes it easy to "look back" at the previous arguments so we can exclude both a flag and + # its value. + return tuple( + argument + for (previous_argument, argument) in zip((None,) + arguments, arguments) + if flag not in (previous_argument, argument) + if not argument.startswith(f'{flag}=') + ) diff --git a/borgmatic/borg/import_key.py b/borgmatic/borg/import_key.py new file mode 100644 index 00000000..9e902778 --- /dev/null +++ b/borgmatic/borg/import_key.py @@ -0,0 +1,70 @@ +import logging +import os + +import borgmatic.config.paths +import borgmatic.logger +from borgmatic.borg import environment, flags +from borgmatic.execute import DO_NOT_CAPTURE, execute_command + +logger = logging.getLogger(__name__) + + +def import_key( + repository_path, + config, + local_borg_version, + import_arguments, + global_arguments, + local_path='borg', + remote_path=None, +): + ''' + Given a local or remote repository path, a configuration dict, the local Borg version, import + arguments, and optional local and remote Borg paths, import the repository key from the + path indicated in the import arguments. + + If the path is empty or "-", then read the key from stdin. + + Raise ValueError if the path is given and it does not exist. + ''' + umask = config.get('umask', None) + lock_wait = config.get('lock_wait', None) + working_directory = borgmatic.config.paths.get_working_directory(config) + + if import_arguments.path and import_arguments.path != '-': + if not os.path.exists(os.path.join(working_directory or '', import_arguments.path)): + raise ValueError(f'Path {import_arguments.path} does not exist. Aborting.') + + input_file = None + else: + input_file = DO_NOT_CAPTURE + + full_command = ( + (local_path, 'key', 'import') + + (('--remote-path', remote_path) if remote_path else ()) + + (('--umask', str(umask)) if umask else ()) + + (('--log-json',) if global_arguments.log_json else ()) + + (('--lock-wait', str(lock_wait)) if lock_wait else ()) + + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + + flags.make_flags('paper', import_arguments.paper) + + flags.make_repository_flags( + repository_path, + local_borg_version, + ) + + ((import_arguments.path,) if input_file is None else ()) + ) + + if global_arguments.dry_run: + logger.info('Skipping key import (dry run)') + return + + execute_command( + full_command, + input_file=input_file, + output_log_level=logging.INFO, + environment=environment.make_environment(config), + working_directory=working_directory, + borg_local_path=local_path, + borg_exit_codes=config.get('borg_exit_codes'), + ) diff --git a/borgmatic/borg/info.py b/borgmatic/borg/info.py index 696048c4..2d90ef52 100644 --- a/borgmatic/borg/info.py +++ b/borgmatic/borg/info.py @@ -48,9 +48,7 @@ def make_info_command( if info_arguments.prefix else ( flags.make_match_archives_flags( - info_arguments.match_archives - or info_arguments.archive - or config.get('match_archives'), + info_arguments.archive or config.get('match_archives'), config.get('archive_name_format'), local_borg_version, ) @@ -102,7 +100,7 @@ def display_archives_info( json_info = execute_command_and_capture_output( json_command, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, borg_exit_codes=borg_exit_codes, @@ -116,7 +114,7 @@ def display_archives_info( execute_command( main_command, output_log_level=logging.ANSWER, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, borg_exit_codes=borg_exit_codes, diff --git a/borgmatic/borg/list.py b/borgmatic/borg/list.py index 755b05c9..fad36e77 100644 --- a/borgmatic/borg/list.py +++ b/borgmatic/borg/list.py @@ -124,7 +124,7 @@ def capture_archive_listing( local_path, remote_path, ), - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), @@ -221,7 +221,7 @@ def list_archive( local_path, remote_path, ), - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=borg_exit_codes, @@ -257,7 +257,7 @@ def list_archive( execute_command( main_command, output_log_level=logging.ANSWER, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=borg_exit_codes, diff --git a/borgmatic/borg/mount.py b/borgmatic/borg/mount.py index 06c6422c..627c42d3 100644 --- a/borgmatic/borg/mount.py +++ b/borgmatic/borg/mount.py @@ -66,7 +66,7 @@ def mount_archive( execute_command( full_command, output_file=DO_NOT_CAPTURE, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), @@ -75,7 +75,7 @@ def mount_archive( execute_command( full_command, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), diff --git a/borgmatic/borg/passcommand.py b/borgmatic/borg/passcommand.py index f70a1392..79132fb4 100644 --- a/borgmatic/borg/passcommand.py +++ b/borgmatic/borg/passcommand.py @@ -9,21 +9,14 @@ logger = logging.getLogger(__name__) @functools.cache -def run_passcommand(passcommand, passphrase_configured, working_directory): +def run_passcommand(passcommand, working_directory): ''' Run the given passcommand using the given working directory and return the passphrase produced - by the command. But bail first if a passphrase is already configured; this mimics Borg's - behavior. + by the command. Cache the results so that the passcommand only needs to run—and potentially prompt the user—once per borgmatic invocation. ''' - if passcommand and passphrase_configured: - logger.warning( - 'Ignoring the "encryption_passcommand" option because "encryption_passphrase" is set' - ) - return None - return borgmatic.execute.execute_command_and_capture_output( shlex.split(passcommand), working_directory=working_directory, @@ -44,7 +37,4 @@ def get_passphrase_from_passcommand(config): if not passcommand: return None - passphrase = config.get('encryption_passphrase') - working_directory = borgmatic.config.paths.get_working_directory(config) - - return run_passcommand(passcommand, bool(passphrase is not None), working_directory) + return run_passcommand(passcommand, borgmatic.config.paths.get_working_directory(config)) diff --git a/borgmatic/borg/pattern.py b/borgmatic/borg/pattern.py index be57bc0e..dfce7c3b 100644 --- a/borgmatic/borg/pattern.py +++ b/borgmatic/borg/pattern.py @@ -20,12 +20,31 @@ class Pattern_style(enum.Enum): PATH_FULL_MATCH = 'pf' +class Pattern_source(enum.Enum): + ''' + Where the pattern came from within borgmatic. This is important because certain use cases (like + filesystem snapshotting) only want to consider patterns that the user actually put in a + configuration file and not patterns from other sources. + ''' + + # The pattern is from a borgmatic configuration option, e.g. listed in "source_directories". + CONFIG = 'config' + + # The pattern is generated internally within borgmatic, e.g. for special file excludes. + INTERNAL = 'internal' + + # The pattern originates from within a borgmatic hook, e.g. a database hook that adds its dump + # directory. + HOOK = 'hook' + + Pattern = collections.namedtuple( 'Pattern', - ('path', 'type', 'style', 'device'), + ('path', 'type', 'style', 'device', 'source'), defaults=( Pattern_type.ROOT, Pattern_style.NONE, None, + Pattern_source.HOOK, ), ) diff --git a/borgmatic/borg/prune.py b/borgmatic/borg/prune.py index b9765fac..34530eeb 100644 --- a/borgmatic/borg/prune.py +++ b/borgmatic/borg/prune.py @@ -41,7 +41,7 @@ def make_prune_flags(config, prune_arguments, local_borg_version): if prefix else ( flags.make_match_archives_flags( - prune_arguments.match_archives or config.get('match_archives'), + config.get('match_archives'), config.get('archive_name_format'), local_borg_version, ) @@ -75,20 +75,26 @@ def prune_archives( + (('--umask', str(umask)) if umask else ()) + (('--log-json',) if global_arguments.log_json else ()) + (('--lock-wait', str(lock_wait)) if lock_wait else ()) - + (('--stats',) if prune_arguments.stats and not dry_run else ()) + + ( + ('--stats',) + if config.get('statistics') + and not dry_run + and not feature.available(feature.Feature.NO_PRUNE_STATS, local_borg_version) + else () + ) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + flags.make_flags_from_arguments( prune_arguments, - excludes=('repository', 'match_archives', 'stats', 'list_archives'), + excludes=('repository', 'match_archives', 'statistics', 'list_details'), ) - + (('--list',) if prune_arguments.list_archives else ()) + + (('--list',) if config.get('list_details') else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (('--dry-run',) if dry_run else ()) + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ()) + flags.make_repository_flags(repository_path, local_borg_version) ) - if prune_arguments.stats or prune_arguments.list_archives: + if config.get('statistics') or config.get('list_details'): output_log_level = logging.ANSWER else: output_log_level = logging.INFO @@ -96,7 +102,7 @@ def prune_archives( execute_command( full_command, output_log_level=output_log_level, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), diff --git a/borgmatic/borg/recreate.py b/borgmatic/borg/recreate.py new file mode 100644 index 00000000..b687256e --- /dev/null +++ b/borgmatic/borg/recreate.py @@ -0,0 +1,103 @@ +import logging +import shlex + +import borgmatic.borg.environment +import borgmatic.borg.feature +import borgmatic.config.paths +import borgmatic.execute +from borgmatic.borg import flags +from borgmatic.borg.create import make_exclude_flags, make_list_filter_flags, write_patterns_file + +logger = logging.getLogger(__name__) + + +def recreate_archive( + repository, + archive, + config, + local_borg_version, + recreate_arguments, + global_arguments, + local_path, + remote_path=None, + patterns=None, +): + ''' + Given a local or remote repository path, an archive name, a configuration dict, the local Borg + version string, an argparse.Namespace of recreate arguments, an argparse.Namespace of global + arguments, optional local and remote Borg paths, executes the recreate command with the given + arguments. + ''' + lock_wait = config.get('lock_wait', None) + exclude_flags = make_exclude_flags(config) + compression = config.get('compression', None) + chunker_params = config.get('chunker_params', None) + # Available recompress MODES: "if-different", "always", "never" (default) + recompress = config.get('recompress', None) + + # Write patterns to a temporary file and use that file with --patterns-from. + patterns_file = write_patterns_file( + patterns, borgmatic.config.paths.get_working_directory(config) + ) + + recreate_command = ( + (local_path, 'recreate') + + (('--remote-path', remote_path) if remote_path else ()) + + (('--log-json',) if global_arguments.log_json else ()) + + (('--lock-wait', str(lock_wait)) if lock_wait is not None else ()) + + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + + (('--patterns-from', patterns_file.name) if patterns_file else ()) + + ( + ( + '--list', + '--filter', + make_list_filter_flags(local_borg_version, global_arguments.dry_run), + ) + if config.get('list_details') + else () + ) + # Flag --target works only for a single archive. + + (('--target', recreate_arguments.target) if recreate_arguments.target and archive else ()) + + ( + ('--comment', shlex.quote(recreate_arguments.comment)) + if recreate_arguments.comment + else () + ) + + (('--timestamp', recreate_arguments.timestamp) if recreate_arguments.timestamp else ()) + + (('--compression', compression) if compression else ()) + + (('--chunker-params', chunker_params) if chunker_params else ()) + + (('--recompress', recompress) if recompress else ()) + + exclude_flags + + ( + ( + flags.make_repository_flags(repository, local_borg_version) + + flags.make_match_archives_flags( + archive or config.get('match_archives'), + config.get('archive_name_format'), + local_borg_version, + ) + ) + if borgmatic.borg.feature.available( + borgmatic.borg.feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version + ) + else ( + flags.make_repository_archive_flags(repository, archive, local_borg_version) + if archive + else flags.make_repository_flags(repository, local_borg_version) + ) + ) + ) + + if global_arguments.dry_run: + logger.info('Skipping the archive recreation (dry run)') + return + + borgmatic.execute.execute_command( + full_command=recreate_command, + output_log_level=logging.INFO, + environment=borgmatic.borg.environment.make_environment(config), + working_directory=borgmatic.config.paths.get_working_directory(config), + borg_local_path=local_path, + borg_exit_codes=config.get('borg_exit_codes'), + ) diff --git a/borgmatic/borg/repo_create.py b/borgmatic/borg/repo_create.py index 7ccf85ba..9ed2619e 100644 --- a/borgmatic/borg/repo_create.py +++ b/borgmatic/borg/repo_create.py @@ -24,7 +24,7 @@ def create_repository( copy_crypt_key=False, append_only=None, storage_quota=None, - make_parent_dirs=False, + make_parent_directories=False, local_path='borg', remote_path=None, ): @@ -79,7 +79,7 @@ def create_repository( + (('--copy-crypt-key',) if copy_crypt_key else ()) + (('--append-only',) if append_only else ()) + (('--storage-quota', storage_quota) if storage_quota else ()) - + (('--make-parent-dirs',) if make_parent_dirs else ()) + + (('--make-parent-dirs',) if make_parent_directories else ()) + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ()) + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ()) + (('--log-json',) if global_arguments.log_json else ()) @@ -98,7 +98,7 @@ def create_repository( execute_command( repo_create_command, output_file=DO_NOT_CAPTURE, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), diff --git a/borgmatic/borg/repo_delete.py b/borgmatic/borg/repo_delete.py index e2241a3e..dee5a1a9 100644 --- a/borgmatic/borg/repo_delete.py +++ b/borgmatic/borg/repo_delete.py @@ -39,14 +39,14 @@ def make_repo_delete_command( + borgmatic.borg.flags.make_flags('umask', config.get('umask')) + borgmatic.borg.flags.make_flags('log-json', global_arguments.log_json) + borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait')) - + borgmatic.borg.flags.make_flags('list', repo_delete_arguments.list_archives) + + borgmatic.borg.flags.make_flags('list', config.get('list_details')) + ( (('--force',) + (('--force',) if repo_delete_arguments.force >= 2 else ())) if repo_delete_arguments.force else () ) + borgmatic.borg.flags.make_flags_from_arguments( - repo_delete_arguments, excludes=('list_archives', 'force', 'repository') + repo_delete_arguments, excludes=('list_details', 'force', 'repository') ) + borgmatic.borg.flags.make_repository_flags(repository['path'], local_borg_version) ) @@ -88,7 +88,7 @@ def delete_repository( if repo_delete_arguments.force or repo_delete_arguments.cache_only else borgmatic.execute.DO_NOT_CAPTURE ), - extra_environment=borgmatic.borg.environment.make_environment(config), + environment=borgmatic.borg.environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), diff --git a/borgmatic/borg/repo_info.py b/borgmatic/borg/repo_info.py index 0c37a052..d3ea2575 100644 --- a/borgmatic/borg/repo_info.py +++ b/borgmatic/borg/repo_info.py @@ -56,7 +56,7 @@ def display_repository_info( if repo_info_arguments.json: return execute_command_and_capture_output( full_command, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, borg_exit_codes=borg_exit_codes, @@ -65,7 +65,7 @@ def display_repository_info( execute_command( full_command, output_log_level=logging.ANSWER, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, borg_exit_codes=borg_exit_codes, diff --git a/borgmatic/borg/repo_list.py b/borgmatic/borg/repo_list.py index f8e46289..9f05adbf 100644 --- a/borgmatic/borg/repo_list.py +++ b/borgmatic/borg/repo_list.py @@ -49,7 +49,7 @@ def resolve_archive_name( output = execute_command_and_capture_output( full_command, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), @@ -113,7 +113,7 @@ def make_repo_list_command( if repo_list_arguments.prefix else ( flags.make_match_archives_flags( - repo_list_arguments.match_archives or config.get('match_archives'), + config.get('match_archives'), config.get('archive_name_format'), local_borg_version, ) @@ -164,7 +164,7 @@ def list_repository( json_listing = execute_command_and_capture_output( json_command, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, borg_exit_codes=borg_exit_codes, @@ -178,7 +178,7 @@ def list_repository( execute_command( main_command, output_log_level=logging.ANSWER, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=working_directory, borg_local_path=local_path, borg_exit_codes=borg_exit_codes, diff --git a/borgmatic/borg/transfer.py b/borgmatic/borg/transfer.py index 473ae0cb..7b168035 100644 --- a/borgmatic/borg/transfer.py +++ b/borgmatic/borg/transfer.py @@ -32,17 +32,22 @@ def transfer_archives( + flags.make_flags('remote-path', remote_path) + flags.make_flags('umask', config.get('umask')) + flags.make_flags('log-json', global_arguments.log_json) - + flags.make_flags('lock-wait', config.get('lock_wait', None)) + + flags.make_flags('lock-wait', config.get('lock_wait')) + + flags.make_flags('progress', config.get('progress')) + ( flags.make_flags_from_arguments( transfer_arguments, - excludes=('repository', 'source_repository', 'archive', 'match_archives'), + excludes=( + 'repository', + 'source_repository', + 'archive', + 'match_archives', + 'progress', + ), ) or ( flags.make_match_archives_flags( - transfer_arguments.match_archives - or transfer_arguments.archive - or config.get('match_archives'), + transfer_arguments.archive or config.get('match_archives'), config.get('archive_name_format'), local_borg_version, ) @@ -56,8 +61,8 @@ def transfer_archives( return execute_command( full_command, output_log_level=logging.ANSWER, - output_file=DO_NOT_CAPTURE if transfer_arguments.progress else None, - extra_environment=environment.make_environment(config), + output_file=DO_NOT_CAPTURE if config.get('progress') else None, + environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), diff --git a/borgmatic/borg/version.py b/borgmatic/borg/version.py index e9cc4035..1c07dce6 100644 --- a/borgmatic/borg/version.py +++ b/borgmatic/borg/version.py @@ -21,7 +21,7 @@ def local_borg_version(config, local_path='borg'): ) output = execute_command_and_capture_output( full_command, - extra_environment=environment.make_environment(config), + environment=environment.make_environment(config), working_directory=borgmatic.config.paths.get_working_directory(config), borg_local_path=local_path, borg_exit_codes=config.get('borg_exit_codes'), diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index e95ec5f4..ec4041cb 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1,8 +1,13 @@ import collections +import io import itertools +import re import sys from argparse import ArgumentParser +import ruamel.yaml + +import borgmatic.config.schema from borgmatic.config import collect ACTION_ALIASES = { @@ -27,6 +32,7 @@ ACTION_ALIASES = { 'break-lock': [], 'key': [], 'borg': [], + 'recreate': [], } @@ -63,9 +69,9 @@ def get_subactions_for_actions(action_parsers): def omit_values_colliding_with_action_names(unparsed_arguments, parsed_arguments): ''' - Given a sequence of string arguments and a dict from action name to parsed argparse.Namespace - arguments, return the string arguments with any values omitted that happen to be the same as - the name of a borgmatic action. + Given unparsed arguments as a sequence of strings and a dict from action name to parsed + argparse.Namespace arguments, return the string arguments with any values omitted that happen to + be the same as the name of a borgmatic action. This prevents, for instance, "check --only extract" from triggering the "extract" action. ''' @@ -282,17 +288,270 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse ) -def make_parsers(): +OMITTED_FLAG_NAMES = {'match-archives', 'progress', 'statistics', 'list-details'} + + +def make_argument_description(schema, flag_name): ''' - Build a global arguments parser, individual action parsers, and a combined parser containing - both. Return them as a tuple. The global parser is useful for parsing just global arguments - while ignoring actions, and the combined parser is handy for displaying help that includes - everything: global flags, a list of actions, etc. + Given a configuration schema dict and a flag name for it, extend the schema's description with + an example or additional information as appropriate based on its type. Return the updated + description for use in a command-line argument. + ''' + description = schema.get('description') + schema_type = schema.get('type') + example = schema.get('example') + pieces = [description] if description else [] + + if '[0]' in flag_name: + pieces.append( + ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).' + ) + + if example and schema_type in ('array', 'object'): + example_buffer = io.StringIO() + yaml = ruamel.yaml.YAML(typ='safe') + yaml.default_flow_style = True + yaml.dump(example, example_buffer) + + pieces.append(f'Example value: "{example_buffer.getvalue().strip()}"') + + return ' '.join(pieces).replace('%', '%%') + + +def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name): + r''' + Given an argparse._ArgumentGroup instance, a sequence of unparsed argument strings, and a dotted + flag name, add command-line array element flags that correspond to the given unparsed arguments. + + Here's the background. We want to support flags that can have arbitrary indices like: + + --foo.bar[1].baz + + But argparse doesn't support that natively because the index can be an arbitrary number. We + won't let that stop us though, will we? + + If the current flag name has an array component in it (e.g. a name with "[0]"), then make a + pattern that would match the flag name regardless of the number that's in it. The idea is that + we want to look for unparsed arguments that appear like the flag name, but instead of "[0]" they + have, say, "[1]" or "[123]". + + Next, we check each unparsed argument against that pattern. If one of them matches, add an + argument flag for it to the argument parser group. Example: + + Let's say flag_name is: + + --foo.bar[0].baz + + ... then the regular expression pattern will be: + + ^--foo\.bar\[\d+\]\.baz + + ... and, if that matches an unparsed argument of: + + --foo.bar[1].baz + + ... then an argument flag will get added equal to that unparsed argument. And so the unparsed + argument will match it when parsing is performed! In this manner, we're using the actual user + CLI input to inform what exact flags we support. + ''' + if '[0]' not in flag_name or not unparsed_arguments or '--help' in unparsed_arguments: + return + + pattern = re.compile(fr'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.")}$') + + try: + # Find an existing list index flag (and its action) corresponding to the given flag name. + (argument_action, existing_flag_name) = next( + (action, action_flag_name) + for action in arguments_group._group_actions + for action_flag_name in action.option_strings + if pattern.match(action_flag_name) + if f'--{flag_name}'.startswith(action_flag_name) + ) + + # Based on the type of the action (e.g. argparse._StoreTrueAction), look up the corresponding + # action registry name (e.g., "store_true") to pass to add_argument(action=...) below. + action_registry_name = next( + registry_name + for registry_name, action_type in arguments_group._registries['action'].items() + # Not using isinstance() here because we only want an exact match—no parent classes. + if type(argument_action) is action_type + ) + except StopIteration: + return + + for unparsed in unparsed_arguments: + unparsed_flag_name = unparsed.split('=', 1)[0] + destination_name = unparsed_flag_name.lstrip('-').replace('-', '_') + + if not pattern.match(unparsed_flag_name) or unparsed_flag_name == existing_flag_name: + continue + + if action_registry_name in ('store_true', 'store_false'): + arguments_group.add_argument( + unparsed_flag_name, + action=action_registry_name, + default=argument_action.default, + dest=destination_name, + required=argument_action.nargs, + ) + else: + arguments_group.add_argument( + unparsed_flag_name, + action=action_registry_name, + choices=argument_action.choices, + default=argument_action.default, + dest=destination_name, + nargs=argument_action.nargs, + required=argument_action.nargs, + type=argument_action.type, + ) + + +def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names=None): + ''' + Given an argparse._ArgumentGroup instance, a configuration schema dict, and a sequence of + unparsed argument strings, convert the entire schema into corresponding command-line flags and + add them to the arguments group. + + For instance, given a schema of: + + { + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'object', + 'properties': { + 'bar': {'type': 'integer'} + } + } + } + } + + ... the following flag will be added to the arguments group: + + --foo.bar + + If "foo" is instead an array of objects, both of the following will get added: + + --foo + --foo[0].bar + + And if names are also passed in, they are considered to be the name components of an option + (e.g. "foo" and "bar") and are used to construct a resulting flag. + + Bail if the schema is not a dict. + ''' + if names is None: + names = () + + if not isinstance(schema, dict): + return + + schema_type = schema.get('type') + + # If this option has multiple types, just use the first one (that isn't "null"). + if isinstance(schema_type, list): + try: + schema_type = next(single_type for single_type in schema_type if single_type != 'null') + except StopIteration: + raise ValueError(f'Unknown type in configuration schema: {schema_type}') + + # If this is an "object" type, recurse for each child option ("property"). + if schema_type == 'object': + properties = schema.get('properties') + + # If there are child properties, recurse for each one. But if there are no child properties, + # fall through so that a flag gets added below for the (empty) object. + if properties: + for name, child in properties.items(): + add_arguments_from_schema( + arguments_group, child, unparsed_arguments, names + (name,) + ) + + return + + # If this is an "array" type, recurse for each items type child option. Don't return yet so that + # a flag also gets added below for the array itself. + if schema_type == 'array': + items = schema.get('items', {}) + properties = borgmatic.config.schema.get_properties(items) + + if properties: + for name, child in properties.items(): + add_arguments_from_schema( + arguments_group, + child, + unparsed_arguments, + names[:-1] + (f'{names[-1]}[0]',) + (name,), + ) + # If there aren't any children, then this is an array of scalars. Recurse accordingly. + else: + add_arguments_from_schema( + arguments_group, items, unparsed_arguments, names[:-1] + (f'{names[-1]}[0]',) + ) + + flag_name = '.'.join(names).replace('_', '-') + + # Certain options already have corresponding flags on individual actions (like "create + # --progress"), so don't bother adding them to the global flags. + if not flag_name or flag_name in OMITTED_FLAG_NAMES: + return + + metavar = names[-1].upper() + description = make_argument_description(schema, flag_name) + + # The object=str and array=str given here is to support specifying an object or an array as a + # YAML string on the command-line. + argument_type = borgmatic.config.schema.parse_type(schema_type, object=str, array=str) + + # As a UX nicety, add separate true and false flags for boolean options. + if schema_type == 'boolean': + arguments_group.add_argument( + f'--{flag_name}', + action='store_true', + default=None, + help=description, + ) + + if names[-1].startswith('no_'): + no_flag_name = '.'.join(names[:-1] + (names[-1][len('no_') :],)).replace('_', '-') + else: + no_flag_name = '.'.join(names[:-1] + ('no-' + names[-1],)).replace('_', '-') + + arguments_group.add_argument( + f'--{no_flag_name}', + dest=flag_name.replace('-', '_'), + action='store_false', + default=None, + help=f'Set the --{flag_name} value to false.', + ) + else: + arguments_group.add_argument( + f'--{flag_name}', + type=argument_type, + metavar=metavar, + help=description, + ) + + add_array_element_arguments(arguments_group, unparsed_arguments, flag_name) + + +def make_parsers(schema, unparsed_arguments): + ''' + Given a configuration schema dict and unparsed arguments as a sequence of strings, build a + global arguments parser, individual action parsers, and a combined parser containing both. + Return them as a tuple. The global parser is useful for parsing just global arguments while + ignoring actions, and the combined parser is handy for displaying help that includes everything: + global flags, a list of actions, etc. ''' config_paths = collect.get_default_config_paths(expand_home=True) unexpanded_config_paths = collect.get_default_config_paths(expand_home=False) - global_parser = ArgumentParser(add_help=False) + # Using allow_abbrev=False here prevents the global parser from erroring about "ambiguous" + # options like --encryption. Such options are intended for an action parser rather than the + # global parser, and so we don't want to error on them here. + global_parser = ArgumentParser(allow_abbrev=False, add_help=False) global_group = global_parser.add_argument_group('global arguments') global_group.add_argument( @@ -309,9 +568,6 @@ def make_parsers(): action='store_true', help='Go through the motions, but do not actually write to any repositories', ) - global_group.add_argument( - '-nc', '--no-color', dest='no_color', action='store_true', help='Disable colored output' - ) global_group.add_argument( '-v', '--verbosity', @@ -388,6 +644,7 @@ def make_parsers(): action='store_true', help='Display installed version number of borgmatic and exit', ) + add_arguments_from_schema(global_group, schema, unparsed_arguments) global_plus_action_parser = ArgumentParser( description=''' @@ -415,7 +672,6 @@ def make_parsers(): '--encryption', dest='encryption_mode', help='Borg repository encryption mode', - required=True, ) repo_create_group.add_argument( '--source-repository', @@ -434,6 +690,7 @@ def make_parsers(): ) repo_create_group.add_argument( '--append-only', + default=None, action='store_true', help='Create an append-only repository', ) @@ -443,6 +700,8 @@ def make_parsers(): ) repo_create_group.add_argument( '--make-parent-dirs', + dest='make_parent_directories', + default=None, action='store_true', help='Create any missing parent directories of the repository directory', ) @@ -477,7 +736,7 @@ def make_parsers(): ) transfer_group.add_argument( '--progress', - default=False, + default=None, action='store_true', help='Display progress as each archive is transferred', ) @@ -544,13 +803,17 @@ def make_parsers(): ) prune_group.add_argument( '--stats', - dest='stats', - default=False, + dest='statistics', + default=None, action='store_true', - help='Display statistics of the pruned archive', + help='Display statistics of the pruned archive [Borg 1 only]', ) prune_group.add_argument( - '--list', dest='list_archives', action='store_true', help='List archives kept/pruned' + '--list', + dest='list_details', + default=None, + action='store_true', + help='List archives kept/pruned', ) prune_group.add_argument( '--oldest', @@ -588,8 +851,7 @@ def make_parsers(): ) compact_group.add_argument( '--progress', - dest='progress', - default=False, + default=None, action='store_true', help='Display progress as each segment is compacted', ) @@ -603,7 +865,7 @@ def make_parsers(): compact_group.add_argument( '--threshold', type=int, - dest='threshold', + dest='compact_threshold', help='Minimum saved space percentage threshold for compacting a segment, defaults to 10', ) compact_group.add_argument( @@ -624,20 +886,24 @@ def make_parsers(): ) create_group.add_argument( '--progress', - dest='progress', - default=False, + default=None, action='store_true', help='Display progress for each file as it is backed up', ) create_group.add_argument( '--stats', - dest='stats', - default=False, + dest='statistics', + default=None, action='store_true', help='Display statistics of archive', ) create_group.add_argument( - '--list', '--files', dest='list_files', action='store_true', help='Show per-file details' + '--list', + '--files', + dest='list_details', + default=None, + action='store_true', + help='Show per-file details', ) create_group.add_argument( '--json', dest='json', default=False, action='store_true', help='Output results as JSON' @@ -658,8 +924,7 @@ def make_parsers(): ) check_group.add_argument( '--progress', - dest='progress', - default=False, + default=None, action='store_true', help='Display progress for each file as it is checked', ) @@ -716,12 +981,15 @@ def make_parsers(): ) delete_group.add_argument( '--list', - dest='list_archives', + dest='list_details', + default=None, action='store_true', help='Show details for the deleted archives', ) delete_group.add_argument( '--stats', + dest='statistics', + default=None, action='store_true', help='Display statistics for the deleted archives', ) @@ -826,8 +1094,7 @@ def make_parsers(): ) extract_group.add_argument( '--progress', - dest='progress', - default=False, + default=None, action='store_true', help='Display progress for each file as it is extracted', ) @@ -902,8 +1169,7 @@ def make_parsers(): ) config_bootstrap_group.add_argument( '--progress', - dest='progress', - default=False, + default=None, action='store_true', help='Display progress for each file as it is extracted', ) @@ -996,7 +1262,12 @@ def make_parsers(): '--tar-filter', help='Name of filter program to pipe data through' ) export_tar_group.add_argument( - '--list', '--files', dest='list_files', action='store_true', help='Show per-file details' + '--list', + '--files', + dest='list_details', + default=None, + action='store_true', + help='Show per-file details', ) export_tar_group.add_argument( '--strip-components', @@ -1107,7 +1378,8 @@ def make_parsers(): ) repo_delete_group.add_argument( '--list', - dest='list_archives', + dest='list_details', + default=None, action='store_true', help='Show details for the archives in the given repository', ) @@ -1479,6 +1751,31 @@ def make_parsers(): '-h', '--help', action='help', help='Show this help message and exit' ) + key_import_parser = key_parsers.add_parser( + 'import', + help='Import a copy of the repository key from backup', + description='Import a copy of the repository key from backup', + add_help=False, + ) + key_import_group = key_import_parser.add_argument_group('key import arguments') + key_import_group.add_argument( + '--paper', + action='store_true', + help='Import interactively from a backup done with --paper', + ) + key_import_group.add_argument( + '--repository', + help='Path of repository to import the key from, defaults to the configured repository if there is only one, quoted globs supported', + ) + key_import_group.add_argument( + '--path', + metavar='PATH', + help='Path to import the key from backup, defaults to stdin', + ) + key_import_group.add_argument( + '-h', '--help', action='help', help='Show this help message and exit' + ) + key_change_passphrase_parser = key_parsers.add_parser( 'change-passphrase', help='Change the passphrase protecting the repository key', @@ -1496,6 +1793,56 @@ def make_parsers(): '-h', '--help', action='help', help='Show this help message and exit' ) + recreate_parser = action_parsers.add_parser( + 'recreate', + aliases=ACTION_ALIASES['recreate'], + help='Recreate an archive in a repository (with Borg 1.2+, you must run compact afterwards to actually free space)', + description='Recreate an archive in a repository (with Borg 1.2+, you must run compact afterwards to actually free space)', + add_help=False, + ) + recreate_group = recreate_parser.add_argument_group('recreate arguments') + recreate_group.add_argument( + '--repository', + help='Path of repository containing archive to recreate, defaults to the configured repository if there is only one, quoted globs supported', + ) + recreate_group.add_argument( + '--archive', + help='Archive name, hash, or series to recreate', + ) + recreate_group.add_argument( + '--list', + dest='list_details', + default=None, + action='store_true', + help='Show per-file details', + ) + recreate_group.add_argument( + '--target', + metavar='TARGET', + help='Create a new archive from the specified archive (via --archive), without replacing it', + ) + recreate_group.add_argument( + '--comment', + metavar='COMMENT', + help='Add a comment text to the archive or, if an archive is not provided, to all matching archives', + ) + recreate_group.add_argument( + '--timestamp', + metavar='TIMESTAMP', + help='Manually override the archive creation date/time (UTC)', + ) + recreate_group.add_argument( + '-a', + '--match-archives', + '--glob-archives', + dest='match_archives', + metavar='PATTERN', + help='Only consider archive names, hashes, or series matching this pattern [Borg 2.x+ only]', + ) + recreate_group.add_argument( + '-h', '--help', action='help', help='Show this help message and exit' + ) + borg_parser = action_parsers.add_parser( 'borg', aliases=ACTION_ALIASES['borg'], @@ -1523,15 +1870,18 @@ def make_parsers(): return global_parser, action_parsers, global_plus_action_parser -def parse_arguments(*unparsed_arguments): +def parse_arguments(schema, *unparsed_arguments): ''' - Given command-line arguments with which this script was invoked, parse the arguments and return - them as a dict mapping from action name (or "global") to an argparse.Namespace instance. + Given a configuration schema dict and the command-line arguments with which this script was + invoked and unparsed arguments as a sequence of strings, parse the arguments and return them as + a dict mapping from action name (or "global") to an argparse.Namespace instance. Raise ValueError if the arguments cannot be parsed. Raise SystemExit with an error code of 0 if "--help" was requested. ''' - global_parser, action_parsers, global_plus_action_parser = make_parsers() + global_parser, action_parsers, global_plus_action_parser = make_parsers( + schema, unparsed_arguments + ) arguments, remaining_action_arguments = parse_arguments_for_actions( unparsed_arguments, action_parsers.choices, global_parser ) @@ -1559,15 +1909,6 @@ def parse_arguments(*unparsed_arguments): f"Unrecognized argument{'s' if len(unknown_arguments) > 1 else ''}: {' '.join(unknown_arguments)}" ) - if 'create' in arguments and arguments['create'].list_files and arguments['create'].progress: - raise ValueError( - 'With the create action, only one of --list (--files) and --progress flags can be used.' - ) - if 'create' in arguments and arguments['create'].list_files and arguments['create'].json: - raise ValueError( - 'With the create action, only one of --list (--files) and --json flags can be used.' - ) - if ( ('list' in arguments and 'repo-info' in arguments and arguments['list'].json) or ('list' in arguments and 'info' in arguments and arguments['list'].json) @@ -1575,15 +1916,6 @@ def parse_arguments(*unparsed_arguments): ): raise ValueError('With the --json flag, multiple actions cannot be used together.') - if ( - 'transfer' in arguments - and arguments['transfer'].archive - and arguments['transfer'].match_archives - ): - raise ValueError( - 'With the transfer action, only one of --archive and --match-archives flags can be used.' - ) - if 'list' in arguments and (arguments['list'].prefix and arguments['list'].match_archives): raise ValueError( 'With the list action, only one of --prefix or --match-archives flags can be used.' diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 8f677191..5bf41db0 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -8,6 +8,8 @@ import time from queue import Queue from subprocess import CalledProcessError +import ruamel.yaml + import borgmatic.actions.borg import borgmatic.actions.break_lock import borgmatic.actions.change_passphrase @@ -21,10 +23,12 @@ import borgmatic.actions.delete import borgmatic.actions.export_key import borgmatic.actions.export_tar import borgmatic.actions.extract +import borgmatic.actions.import_key import borgmatic.actions.info import borgmatic.actions.list import borgmatic.actions.mount import borgmatic.actions.prune +import borgmatic.actions.recreate import borgmatic.actions.repo_create import borgmatic.actions.repo_delete import borgmatic.actions.repo_info @@ -33,6 +37,8 @@ import borgmatic.actions.restore import borgmatic.actions.transfer import borgmatic.commands.completion.bash import borgmatic.commands.completion.fish +import borgmatic.config.load +import borgmatic.config.paths from borgmatic.borg import umount as borg_umount from borgmatic.borg import version as borg_version from borgmatic.commands.arguments import parse_arguments @@ -67,6 +73,113 @@ def get_skip_actions(config, arguments): return skip_actions +class Monitoring_hooks: + ''' + A Python context manager for pinging monitoring hooks for the start state before the wrapped + code and log and finish (or failure) after the wrapped code. Also responsible for + initializing/destroying the monitoring hooks. + + Example use as a context manager: + + with Monitoring_hooks(config_filename, config, arguments, global_arguments): + do_stuff() + ''' + + def __init__(self, config_filename, config, arguments, global_arguments): + ''' + Given a configuration filename, a configuration dict, command-line arguments as an + argparse.Namespace, and global arguments as an argparse.Namespace, save relevant data points + for use below. + ''' + using_primary_action = {'create', 'prune', 'compact', 'check'}.intersection(arguments) + self.config_filename = config_filename + self.config = config + self.dry_run = global_arguments.dry_run + self.monitoring_log_level = verbosity_to_log_level(global_arguments.monitoring_verbosity) + self.monitoring_hooks_are_activated = ( + using_primary_action and self.monitoring_log_level != DISABLED + ) + + def __enter__(self): + ''' + If monitoring hooks are enabled and a primary action is in use, initialize monitoring hooks + and ping them for the "start" state. + ''' + if not self.monitoring_hooks_are_activated: + return + + dispatch.call_hooks( + 'initialize_monitor', + self.config, + dispatch.Hook_type.MONITORING, + self.config_filename, + self.monitoring_log_level, + self.dry_run, + ) + + try: + dispatch.call_hooks( + 'ping_monitor', + self.config, + dispatch.Hook_type.MONITORING, + self.config_filename, + monitor.State.START, + self.monitoring_log_level, + self.dry_run, + ) + except (OSError, CalledProcessError) as error: + raise ValueError(f'Error pinging monitor: {error}') + + def __exit__(self, exception_type, exception, traceback): + ''' + If monitoring hooks are enabled and a primary action is in use, ping monitoring hooks for + the "log" state and also the "finish" or "fail" states (depending on whether there's an + exception). Lastly, destroy monitoring hooks. + ''' + if not self.monitoring_hooks_are_activated: + return + + # Send logs irrespective of error. + try: + dispatch.call_hooks( + 'ping_monitor', + self.config, + dispatch.Hook_type.MONITORING, + self.config_filename, + monitor.State.LOG, + self.monitoring_log_level, + self.dry_run, + ) + except (OSError, CalledProcessError) as error: + raise ValueError(f'Error pinging monitor: {error}') + + try: + dispatch.call_hooks( + 'ping_monitor', + self.config, + dispatch.Hook_type.MONITORING, + self.config_filename, + monitor.State.FAIL if exception else monitor.State.FINISH, + self.monitoring_log_level, + self.dry_run, + ) + except (OSError, CalledProcessError) as error: + # If the wrapped code errored, prefer raising that exception, as it's probably more + # important than a monitor failing to ping. + if exception: + return + + raise ValueError(f'Error pinging monitor: {error}') + + dispatch.call_hooks( + 'destroy_monitor', + self.config, + dispatch.Hook_type.MONITORING, + self.monitoring_log_level, + self.dry_run, + ) + + def run_configuration(config_filename, config, config_paths, arguments): ''' Given a config filename, the corresponding parsed config dict, a sequence of loaded @@ -84,11 +197,9 @@ def run_configuration(config_filename, config, config_paths, arguments): remote_path = config.get('remote_path') retries = config.get('retries', 0) retry_wait = config.get('retry_wait', 0) + repo_queue = Queue() encountered_error = None - error_repository = '' - using_primary_action = {'create', 'prune', 'compact', 'check'}.intersection(arguments) - monitoring_log_level = verbosity_to_log_level(global_arguments.monitoring_verbosity) - monitoring_hooks_are_activated = using_primary_action and monitoring_log_level != DISABLED + error_repository = None skip_actions = get_skip_actions(config, arguments) if skip_actions: @@ -97,168 +208,105 @@ def run_configuration(config_filename, config, config_paths, arguments): ) try: - local_borg_version = borg_version.local_borg_version(config, local_path) - logger.debug(f'Borg {local_borg_version}') + with Monitoring_hooks(config_filename, config, arguments, global_arguments): + with borgmatic.hooks.command.Before_after_hooks( + command_hooks=config.get('commands'), + before_after='configuration', + umask=config.get('umask'), + working_directory=borgmatic.config.paths.get_working_directory(config), + dry_run=global_arguments.dry_run, + action_names=arguments.keys(), + configuration_filename=config_filename, + log_file=arguments['global'].log_file or '', + ): + try: + local_borg_version = borg_version.local_borg_version(config, local_path) + logger.debug(f'Borg {local_borg_version}') + except (OSError, CalledProcessError, ValueError) as error: + yield from log_error_records( + f'{config_filename}: Error getting local Borg version', error + ) + return + + for repo in config['repositories']: + repo_queue.put( + (repo, 0), + ) + + while not repo_queue.empty(): + repository, retry_num = repo_queue.get() + + with Log_prefix(repository.get('label', repository['path'])): + logger.debug('Running actions for repository') + timeout = retry_num * retry_wait + if timeout: + logger.warning(f'Sleeping {timeout}s before next retry') + time.sleep(timeout) + try: + yield from run_actions( + arguments=arguments, + config_filename=config_filename, + config=config, + config_paths=config_paths, + local_path=local_path, + remote_path=remote_path, + local_borg_version=local_borg_version, + repository=repository, + ) + except (OSError, CalledProcessError, ValueError) as error: + if retry_num < retries: + repo_queue.put( + (repository, retry_num + 1), + ) + tuple( # Consume the generator so as to trigger logging. + log_error_records( + 'Error running actions for repository', + error, + levelno=logging.WARNING, + log_command_error_output=True, + ) + ) + logger.warning(f'Retrying... attempt {retry_num + 1}/{retries}') + continue + + if command.considered_soft_failure(error): + continue + + yield from log_error_records( + 'Error running actions for repository', + error, + ) + encountered_error = error + error_repository = repository + except (OSError, CalledProcessError, ValueError) as error: - yield from log_error_records(f'{config_filename}: Error getting local Borg version', error) + yield from log_error_records('Error running configuration', error) + + encountered_error = error + + if not encountered_error: return try: - if monitoring_hooks_are_activated: - dispatch.call_hooks( - 'initialize_monitor', - config, - dispatch.Hook_type.MONITORING, - config_filename, - monitoring_log_level, - global_arguments.dry_run, - ) - - dispatch.call_hooks( - 'ping_monitor', - config, - dispatch.Hook_type.MONITORING, - config_filename, - monitor.State.START, - monitoring_log_level, - global_arguments.dry_run, - ) + command.execute_hooks( + command.filter_hooks( + config.get('commands'), after='error', action_names=arguments.keys() + ), + config.get('umask'), + borgmatic.config.paths.get_working_directory(config), + global_arguments.dry_run, + configuration_filename=config_filename, + log_file=arguments['global'].log_file or '', + repository=error_repository.get('path', '') if error_repository else '', + repository_label=error_repository.get('label', '') if error_repository else '', + error=encountered_error, + output=getattr(encountered_error, 'output', ''), + ) except (OSError, CalledProcessError) as error: if command.considered_soft_failure(error): return - encountered_error = error - yield from log_error_records(f'{config_filename}: Error pinging monitor', error) - - if not encountered_error: - repo_queue = Queue() - for repo in config['repositories']: - repo_queue.put( - (repo, 0), - ) - - while not repo_queue.empty(): - repository, retry_num = repo_queue.get() - - with Log_prefix(repository.get('label', repository['path'])): - logger.debug('Running actions for repository') - timeout = retry_num * retry_wait - if timeout: - logger.warning(f'Sleeping {timeout}s before next retry') - time.sleep(timeout) - try: - yield from run_actions( - arguments=arguments, - config_filename=config_filename, - config=config, - config_paths=config_paths, - local_path=local_path, - remote_path=remote_path, - local_borg_version=local_borg_version, - repository=repository, - ) - except (OSError, CalledProcessError, ValueError) as error: - if retry_num < retries: - repo_queue.put( - (repository, retry_num + 1), - ) - tuple( # Consume the generator so as to trigger logging. - log_error_records( - 'Error running actions for repository', - error, - levelno=logging.WARNING, - log_command_error_output=True, - ) - ) - logger.warning(f'Retrying... attempt {retry_num + 1}/{retries}') - continue - - if command.considered_soft_failure(error): - continue - - yield from log_error_records( - 'Error running actions for repository', - error, - ) - encountered_error = error - error_repository = repository['path'] - - try: - if monitoring_hooks_are_activated: - # Send logs irrespective of error. - dispatch.call_hooks( - 'ping_monitor', - config, - dispatch.Hook_type.MONITORING, - config_filename, - monitor.State.LOG, - monitoring_log_level, - global_arguments.dry_run, - ) - except (OSError, CalledProcessError) as error: - if not command.considered_soft_failure(error): - encountered_error = error - yield from log_error_records('Error pinging monitor', error) - - if not encountered_error: - try: - if monitoring_hooks_are_activated: - dispatch.call_hooks( - 'ping_monitor', - config, - dispatch.Hook_type.MONITORING, - config_filename, - monitor.State.FINISH, - monitoring_log_level, - global_arguments.dry_run, - ) - dispatch.call_hooks( - 'destroy_monitor', - config, - dispatch.Hook_type.MONITORING, - monitoring_log_level, - global_arguments.dry_run, - ) - except (OSError, CalledProcessError) as error: - if command.considered_soft_failure(error): - return - - encountered_error = error - yield from log_error_records(f'{config_filename}: Error pinging monitor', error) - - if encountered_error and using_primary_action: - try: - command.execute_hook( - config.get('on_error'), - config.get('umask'), - config_filename, - 'on-error', - global_arguments.dry_run, - repository=error_repository, - error=encountered_error, - output=getattr(encountered_error, 'output', ''), - ) - dispatch.call_hooks( - 'ping_monitor', - config, - dispatch.Hook_type.MONITORING, - config_filename, - monitor.State.FAIL, - monitoring_log_level, - global_arguments.dry_run, - ) - dispatch.call_hooks( - 'destroy_monitor', - config, - dispatch.Hook_type.MONITORING, - monitoring_log_level, - global_arguments.dry_run, - ) - except (OSError, CalledProcessError) as error: - if command.considered_soft_failure(error): - return - - yield from log_error_records(f'{config_filename}: Error running on-error hook', error) + yield from log_error_records(f'{config_filename}: Error running after error hook', error) def run_actions( @@ -289,6 +337,7 @@ def run_actions( global_arguments = arguments['global'] dry_run_label = ' (dry run; not making any changes)' if global_arguments.dry_run else '' hook_context = { + 'configuration_filename': config_filename, 'repository_label': repository.get('label', ''), 'log_file': global_arguments.log_file if global_arguments.log_file else '', # Deprecated: For backwards compatibility with borgmatic < 1.6.0. @@ -297,251 +346,272 @@ def run_actions( } skip_actions = set(get_skip_actions(config, arguments)) - command.execute_hook( - config.get('before_actions'), - config.get('umask'), - config_filename, - 'pre-actions', - global_arguments.dry_run, + with borgmatic.hooks.command.Before_after_hooks( + command_hooks=config.get('commands'), + before_after='repository', + umask=config.get('umask'), + working_directory=borgmatic.config.paths.get_working_directory(config), + dry_run=global_arguments.dry_run, + action_names=arguments.keys(), **hook_context, - ) + ): + for action_name, action_arguments in arguments.items(): + if action_name == 'global': + continue - for action_name, action_arguments in arguments.items(): - if action_name == 'repo-create' and action_name not in skip_actions: - borgmatic.actions.repo_create.run_repo_create( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'transfer' and action_name not in skip_actions: - borgmatic.actions.transfer.run_transfer( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'create' and action_name not in skip_actions: - yield from borgmatic.actions.create.run_create( - config_filename, - repository, - config, - config_paths, - hook_context, - local_borg_version, - action_arguments, - global_arguments, - dry_run_label, - local_path, - remote_path, - ) - elif action_name == 'prune' and action_name not in skip_actions: - borgmatic.actions.prune.run_prune( - config_filename, - repository, - config, - hook_context, - local_borg_version, - action_arguments, - global_arguments, - dry_run_label, - local_path, - remote_path, - ) - elif action_name == 'compact' and action_name not in skip_actions: - borgmatic.actions.compact.run_compact( - config_filename, - repository, - config, - hook_context, - local_borg_version, - action_arguments, - global_arguments, - dry_run_label, - local_path, - remote_path, - ) - elif action_name == 'check' and action_name not in skip_actions: - if checks.repository_enabled_for_checks(repository, config): - borgmatic.actions.check.run_check( - config_filename, - repository, - config, - hook_context, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'extract' and action_name not in skip_actions: - borgmatic.actions.extract.run_extract( - config_filename, - repository, - config, - hook_context, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'export-tar' and action_name not in skip_actions: - borgmatic.actions.export_tar.run_export_tar( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'mount' and action_name not in skip_actions: - borgmatic.actions.mount.run_mount( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'restore' and action_name not in skip_actions: - borgmatic.actions.restore.run_restore( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'repo-list' and action_name not in skip_actions: - yield from borgmatic.actions.repo_list.run_repo_list( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'list' and action_name not in skip_actions: - yield from borgmatic.actions.list.run_list( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'repo-info' and action_name not in skip_actions: - yield from borgmatic.actions.repo_info.run_repo_info( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'info' and action_name not in skip_actions: - yield from borgmatic.actions.info.run_info( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'break-lock' and action_name not in skip_actions: - borgmatic.actions.break_lock.run_break_lock( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'export' and action_name not in skip_actions: - borgmatic.actions.export_key.run_export_key( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'change-passphrase' and action_name not in skip_actions: - borgmatic.actions.change_passphrase.run_change_passphrase( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'delete' and action_name not in skip_actions: - borgmatic.actions.delete.run_delete( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'repo-delete' and action_name not in skip_actions: - borgmatic.actions.repo_delete.run_repo_delete( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - elif action_name == 'borg' and action_name not in skip_actions: - borgmatic.actions.borg.run_borg( - repository, - config, - local_borg_version, - action_arguments, - global_arguments, - local_path, - remote_path, - ) - - command.execute_hook( - config.get('after_actions'), - config.get('umask'), - config_filename, - 'post-actions', - global_arguments.dry_run, - **hook_context, - ) + with borgmatic.hooks.command.Before_after_hooks( + command_hooks=config.get('commands'), + before_after='action', + umask=config.get('umask'), + working_directory=borgmatic.config.paths.get_working_directory(config), + dry_run=global_arguments.dry_run, + action_names=arguments.keys(), + **hook_context, + ): + if action_name == 'repo-create' and action_name not in skip_actions: + borgmatic.actions.repo_create.run_repo_create( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'transfer' and action_name not in skip_actions: + borgmatic.actions.transfer.run_transfer( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'create' and action_name not in skip_actions: + yield from borgmatic.actions.create.run_create( + config_filename, + repository, + config, + config_paths, + local_borg_version, + action_arguments, + global_arguments, + dry_run_label, + local_path, + remote_path, + ) + elif action_name == 'recreate' and action_name not in skip_actions: + borgmatic.actions.recreate.run_recreate( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'prune' and action_name not in skip_actions: + borgmatic.actions.prune.run_prune( + config_filename, + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + dry_run_label, + local_path, + remote_path, + ) + elif action_name == 'compact' and action_name not in skip_actions: + borgmatic.actions.compact.run_compact( + config_filename, + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + dry_run_label, + local_path, + remote_path, + ) + elif action_name == 'check' and action_name not in skip_actions: + if checks.repository_enabled_for_checks(repository, config): + borgmatic.actions.check.run_check( + config_filename, + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'extract' and action_name not in skip_actions: + borgmatic.actions.extract.run_extract( + config_filename, + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'export-tar' and action_name not in skip_actions: + borgmatic.actions.export_tar.run_export_tar( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'mount' and action_name not in skip_actions: + borgmatic.actions.mount.run_mount( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'restore' and action_name not in skip_actions: + borgmatic.actions.restore.run_restore( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'repo-list' and action_name not in skip_actions: + yield from borgmatic.actions.repo_list.run_repo_list( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'list' and action_name not in skip_actions: + yield from borgmatic.actions.list.run_list( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'repo-info' and action_name not in skip_actions: + yield from borgmatic.actions.repo_info.run_repo_info( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'info' and action_name not in skip_actions: + yield from borgmatic.actions.info.run_info( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'break-lock' and action_name not in skip_actions: + borgmatic.actions.break_lock.run_break_lock( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'export' and action_name not in skip_actions: + borgmatic.actions.export_key.run_export_key( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'import' and action_name not in skip_actions: + borgmatic.actions.import_key.run_import_key( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'change-passphrase' and action_name not in skip_actions: + borgmatic.actions.change_passphrase.run_change_passphrase( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'delete' and action_name not in skip_actions: + borgmatic.actions.delete.run_delete( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'repo-delete' and action_name not in skip_actions: + borgmatic.actions.repo_delete.run_repo_delete( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) + elif action_name == 'borg' and action_name not in skip_actions: + borgmatic.actions.borg.run_borg( + repository, + config, + local_borg_version, + action_arguments, + global_arguments, + local_path, + remote_path, + ) -def load_configurations(config_filenames, overrides=None, resolve_env=True): +def load_configurations(config_filenames, arguments, overrides=None, resolve_env=True): ''' - Given a sequence of configuration filenames, load and validate each configuration file. Return - the results as a tuple of: dict of configuration filename to corresponding parsed configuration, - a sequence of paths for all loaded configuration files (including includes), and a sequence of - logging.LogRecord instances containing any parse errors. + Given a sequence of configuration filenames, arguments as a dict from action name to + argparse.Namespace, a sequence of configuration file override strings in the form of + "option.suboption=value", and whether to resolve environment variables, load and validate each + configuration file. Return the results as a tuple of: dict of configuration filename to + corresponding parsed configuration, a sequence of paths for all loaded configuration files + (including includes), and a sequence of logging.LogRecord instances containing any parse errors. Log records are returned here instead of being logged directly because logging isn't yet - initialized at this point! + initialized at this point! (Although with the Delayed_logging_handler now in place, maybe this + approach could change.) ''' # Dict mapping from config filename to corresponding parsed config dict. configs = collections.OrderedDict() @@ -563,7 +633,11 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True): ) try: configs[config_filename], paths, parse_logs = validate.parse_configuration( - config_filename, validate.schema_filename(), overrides, resolve_env + config_filename, + validate.schema_filename(), + arguments, + overrides, + resolve_env, ) config_paths.update(paths) logs.extend(parse_logs) @@ -804,19 +878,21 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): ) return - if 'create' in arguments: - try: - for config_filename, config in configs.items(): - command.execute_hook( - config.get('before_everything'), - config.get('umask'), - config_filename, - 'pre-everything', - arguments['global'].dry_run, - ) - except (CalledProcessError, ValueError, OSError) as error: - yield from log_error_records('Error running pre-everything hook', error) - return + try: + for config_filename, config in configs.items(): + command.execute_hooks( + command.filter_hooks( + config.get('commands'), before='everything', action_names=arguments.keys() + ), + config.get('umask'), + borgmatic.config.paths.get_working_directory(config), + arguments['global'].dry_run, + configuration_filename=config_filename, + log_file=arguments['global'].log_file or '', + ) + except (CalledProcessError, ValueError, OSError) as error: + yield from log_error_records('Error running before everything hook', error) + return # Execute the actions corresponding to each configuration file. json_results = [] @@ -824,6 +900,7 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): for config_filename, config in configs.items(): with Log_prefix(config_filename): results = list(run_configuration(config_filename, config, config_paths, arguments)) + error_logs = tuple( result for result in results if isinstance(result, logging.LogRecord) ) @@ -836,7 +913,7 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): dict( levelno=logging.INFO, levelname='INFO', - msg='Successfully ran configuration file', + msg=f'{config_filename}: Successfully ran configuration file', ) ) if results: @@ -856,18 +933,20 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): if json_results: sys.stdout.write(json.dumps(json_results)) - if 'create' in arguments: - try: - for config_filename, config in configs.items(): - command.execute_hook( - config.get('after_everything'), - config.get('umask'), - config_filename, - 'post-everything', - arguments['global'].dry_run, - ) - except (CalledProcessError, ValueError, OSError) as error: - yield from log_error_records('Error running post-everything hook', error) + try: + for config_filename, config in configs.items(): + command.execute_hooks( + command.filter_hooks( + config.get('commands'), after='everything', action_names=arguments.keys() + ), + config.get('umask'), + borgmatic.config.paths.get_working_directory(config), + arguments['global'].dry_run, + configuration_filename=config_filename, + log_file=arguments['global'].log_file or '', + ) + except (CalledProcessError, ValueError, OSError) as error: + yield from log_error_records('Error running after everything hook', error) def exit_with_help_link(): # pragma: no cover @@ -879,12 +958,33 @@ def exit_with_help_link(): # pragma: no cover sys.exit(1) +def check_and_show_help_on_no_args(configs): + ''' + Check if the borgmatic command is run without any arguments. If the configuration option + "default_actions" is set to False, show the help message. Otherwise, trigger the default backup + behavior. + ''' + if len(sys.argv) == 1: # No arguments provided + default_actions = any(config.get('default_actions', True) for config in configs.values()) + if not default_actions: + parse_arguments('--help') + sys.exit(0) + + def main(extra_summary_logs=[]): # pragma: no cover configure_signals() configure_delayed_logging() + schema_filename = validate.schema_filename() try: - arguments = parse_arguments(*sys.argv[1:]) + schema = borgmatic.config.load.load_configuration(schema_filename) + except (ruamel.yaml.error.YAMLError, RecursionError) as error: + configure_logging(logging.CRITICAL) + logger.critical(error) + exit_with_help_link() + + try: + arguments = parse_arguments(schema, *sys.argv[1:]) except ValueError as error: configure_logging(logging.CRITICAL) logger.critical(error) @@ -909,8 +1009,15 @@ def main(extra_summary_logs=[]): # pragma: no cover config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths)) configs, config_paths, parse_logs = load_configurations( - config_filenames, global_arguments.overrides, global_arguments.resolve_env + config_filenames, + arguments, + global_arguments.overrides, + resolve_env=global_arguments.resolve_env and not validate, ) + + # Use the helper function to check and show help on no arguments, passing the preloaded configs + check_and_show_help_on_no_args(configs) + configuration_parse_errors = ( (max(log.levelno for log in parse_logs) >= logging.CRITICAL) if parse_logs else False ) @@ -918,7 +1025,7 @@ def main(extra_summary_logs=[]): # pragma: no cover any_json_flags = any( getattr(sub_arguments, 'json', False) for sub_arguments in arguments.values() ) - color_enabled = should_do_markup(global_arguments.no_color or any_json_flags, configs) + color_enabled = should_do_markup(configs, any_json_flags) try: configure_logging( diff --git a/borgmatic/commands/completion/bash.py b/borgmatic/commands/completion/bash.py index 7bf28a42..c72fbbec 100644 --- a/borgmatic/commands/completion/bash.py +++ b/borgmatic/commands/completion/bash.py @@ -1,5 +1,7 @@ import borgmatic.commands.arguments import borgmatic.commands.completion.actions +import borgmatic.commands.completion.flag +import borgmatic.config.validate def parser_flags(parser): @@ -7,7 +9,12 @@ def parser_flags(parser): Given an argparse.ArgumentParser instance, return its argument flags in a space-separated string. ''' - return ' '.join(option for action in parser._actions for option in action.option_strings) + return ' '.join( + flag_variant + for action in parser._actions + for flag_name in action.option_strings + for flag_variant in borgmatic.commands.completion.flag.variants(flag_name) + ) def bash_completion(): @@ -19,7 +26,10 @@ def bash_completion(): unused_global_parser, action_parsers, global_plus_action_parser, - ) = borgmatic.commands.arguments.make_parsers() + ) = borgmatic.commands.arguments.make_parsers( + schema=borgmatic.config.validate.load_schema(borgmatic.config.validate.schema_filename()), + unparsed_arguments=(), + ) global_flags = parser_flags(global_plus_action_parser) # Avert your eyes. diff --git a/borgmatic/commands/completion/fish.py b/borgmatic/commands/completion/fish.py index edca0226..31b83e9c 100644 --- a/borgmatic/commands/completion/fish.py +++ b/borgmatic/commands/completion/fish.py @@ -4,6 +4,7 @@ from textwrap import dedent import borgmatic.commands.arguments import borgmatic.commands.completion.actions +import borgmatic.config.validate def has_file_options(action: Action): @@ -26,9 +27,11 @@ def has_choice_options(action: Action): def has_unknown_required_param_options(action: Action): ''' A catch-all for options that take a required parameter, but we don't know what the parameter is. - This should be used last. These are actions that take something like a glob, a list of numbers, or a string. + This should be used last. These are actions that take something like a glob, a list of numbers, + or a string. - Actions that match this pattern should not show the normal arguments, because those are unlikely to be valid. + Actions that match this pattern should not show the normal arguments, because those are unlikely + to be valid. ''' return ( action.required is True @@ -52,9 +55,9 @@ def has_exact_options(action: Action): def exact_options_completion(action: Action): ''' - Given an argparse.Action instance, return a completion invocation that forces file completions, options completion, - or just that some value follow the action, if the action takes such an argument and was the last action on the - command line prior to the cursor. + Given an argparse.Action instance, return a completion invocation that forces file completions, + options completion, or just that some value follow the action, if the action takes such an + argument and was the last action on the command line prior to the cursor. Otherwise, return an empty string. ''' @@ -80,8 +83,9 @@ def exact_options_completion(action: Action): def dedent_strip_as_tuple(string: str): ''' - Dedent a string, then strip it to avoid requiring your first line to have content, then return a tuple of the string. - Makes it easier to write multiline strings for completions when you join them with a tuple. + Dedent a string, then strip it to avoid requiring your first line to have content, then return a + tuple of the string. Makes it easier to write multiline strings for completions when you join + them with a tuple. ''' return (dedent(string).strip('\n'),) @@ -95,7 +99,10 @@ def fish_completion(): unused_global_parser, action_parsers, global_plus_action_parser, - ) = borgmatic.commands.arguments.make_parsers() + ) = borgmatic.commands.arguments.make_parsers( + schema=borgmatic.config.validate.load_schema(borgmatic.config.validate.schema_filename()), + unparsed_arguments=(), + ) all_action_parsers = ' '.join(action for action in action_parsers.choices.keys()) diff --git a/borgmatic/commands/completion/flag.py b/borgmatic/commands/completion/flag.py new file mode 100644 index 00000000..5a6bc797 --- /dev/null +++ b/borgmatic/commands/completion/flag.py @@ -0,0 +1,13 @@ +def variants(flag_name): + ''' + Given a flag name as a string, yield it and any variations that should be complete-able as well. + For instance, for a string like "--foo[0].bar", yield "--foo[0].bar", "--foo[1].bar", ..., + "--foo[9].bar". + ''' + if '[0]' in flag_name: + for index in range(0, 10): + yield flag_name.replace('[0]', f'[{index}]') + + return + + yield flag_name diff --git a/borgmatic/config/arguments.py b/borgmatic/config/arguments.py new file mode 100644 index 00000000..d5996f0a --- /dev/null +++ b/borgmatic/config/arguments.py @@ -0,0 +1,176 @@ +import io +import re + +import ruamel.yaml + +import borgmatic.config.schema + +LIST_INDEX_KEY_PATTERN = re.compile(r'^(?P[a-zA-z-]+)\[(?P\d+)\]$') + + +def set_values(config, keys, value): + ''' + Given a configuration dict, a sequence of parsed key strings, and a string value, descend into + the configuration hierarchy based on the given keys and set the value into the right place. + For example, consider these keys: + + ('foo', 'bar', 'baz') + + This looks up "foo" in the given configuration dict. And within that, it looks up "bar". And + then within that, it looks up "baz" and sets it to the given value. Another example: + + ('mylist[0]', 'foo') + + This looks for the zeroth element of "mylist" in the given configuration. And within that, it + looks up "foo" and sets it to the given value. + ''' + if not keys: + return + + first_key = keys[0] + + # Support "mylist[0]" list index syntax. + match = LIST_INDEX_KEY_PATTERN.match(first_key) + + if match: + list_key = match.group('list_name') + list_index = int(match.group('index')) + + try: + if len(keys) == 1: + config[list_key][list_index] = value + + return + + if list_key not in config: + config[list_key] = [] + + set_values(config[list_key][list_index], keys[1:], value) + except (IndexError, KeyError): + raise ValueError(f'Argument list index {first_key} is out of range') + + return + + if len(keys) == 1: + config[first_key] = value + + return + + if first_key not in config: + config[first_key] = {} + + set_values(config[first_key], keys[1:], value) + + +def type_for_option(schema, option_keys): + ''' + Given a configuration schema dict and a sequence of keys identifying a potentially nested + option, e.g. ('extra_borg_options', 'create'), return the schema type of that option as a + string. + + Return None if the option or its type cannot be found in the schema. + ''' + option_schema = schema + + for key in option_keys: + # Support "name[0]"-style list index syntax. + match = LIST_INDEX_KEY_PATTERN.match(key) + properties = borgmatic.config.schema.get_properties(option_schema) + + try: + if match: + option_schema = properties[match.group('list_name')]['items'] + else: + option_schema = properties[key] + except KeyError: + return None + + try: + return option_schema['type'] + except KeyError: + return None + + +def convert_value_type(value, option_type): + ''' + Given a string value and its schema type as a string, determine its logical type (string, + boolean, integer, etc.), and return it converted to that type. + + If the destination option type is a string, then leave the value as-is so that special + characters in it don't get interpreted as YAML during conversion. + + And if the source value isn't a string, return it as-is. + + Raise ruamel.yaml.error.YAMLError if there's a parse issue with the YAML. + Raise ValueError if the parsed value doesn't match the option type. + ''' + if not isinstance(value, str): + return value + + if option_type == 'string': + return value + + try: + parsed_value = ruamel.yaml.YAML(typ='safe').load(io.StringIO(value)) + except ruamel.yaml.error.YAMLError as error: + raise ValueError(f'Argument value "{value}" is invalid: {error.problem}') + + if not isinstance(parsed_value, borgmatic.config.schema.parse_type(option_type)): + raise ValueError(f'Argument value "{value}" is not of the expected type: {option_type}') + + return parsed_value + + +def prepare_arguments_for_config(global_arguments, schema): + ''' + Given global arguments as an argparse.Namespace and a configuration schema dict, parse each + argument that corresponds to an option in the schema and return a sequence of tuples (keys, + values) for that option, where keys is a sequence of strings. For instance, given the following + arguments: + + argparse.Namespace(**{'my_option.sub_option': 'value1', 'other_option': 'value2'}) + + ... return this: + + ( + (('my_option', 'sub_option'), 'value1'), + (('other_option',), 'value2'), + ) + ''' + prepared_values = [] + + for argument_name, value in global_arguments.__dict__.items(): + if value is None: + continue + + keys = tuple(argument_name.split('.')) + option_type = type_for_option(schema, keys) + + # The argument doesn't correspond to any option in the schema, so ignore it. It's + # probably a flag that borgmatic has on the command-line but not in configuration. + if option_type is None: + continue + + prepared_values.append( + ( + keys, + convert_value_type(value, option_type), + ) + ) + + return tuple(prepared_values) + + +def apply_arguments_to_config(config, schema, arguments): + ''' + Given a configuration dict, a corresponding configuration schema dict, and arguments as a dict + from action name to argparse.Namespace, set those given argument values into their corresponding + configuration options in the configuration dict. + + This supports argument flags of the from "--foo.bar.baz" where each dotted component is a nested + configuration object. Additionally, flags like "--foo.bar[0].baz" are supported to update a list + element in the configuration. + ''' + for action_arguments in arguments.values(): + for keys, value in prepare_arguments_for_config(action_arguments, schema): + set_values(config, keys, value) diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index f42c1690..734457f2 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -5,6 +5,7 @@ import re import ruamel.yaml +import borgmatic.config.schema from borgmatic.config import load, normalize INDENT = 4 @@ -21,45 +22,59 @@ def insert_newline_before_comment(config, field_name): ) -def get_properties(schema): - ''' - Given a schema dict, return its properties. But if it's got sub-schemas with multiple different - potential properties, returned their merged properties instead. - ''' - if 'oneOf' in schema: - return dict( - collections.ChainMap(*[sub_schema['properties'] for sub_schema in schema['oneOf']]) - ) - - return schema['properties'] +SCALAR_SCHEMA_TYPES = {'string', 'boolean', 'integer', 'number'} -def schema_to_sample_configuration(schema, level=0, parent_is_sequence=False): +def schema_to_sample_configuration(schema, source_config=None, level=0, parent_is_sequence=False): ''' - Given a loaded configuration schema, generate and return sample config for it. Include comments - for each option based on the schema "description". + Given a loaded configuration schema and a source configuration, generate and return sample + config for the schema. Include comments for each option based on the schema "description". + + If a source config is given, walk it alongside the given schema so that both can be taken into + account when commenting out particular options in add_comments_to_configuration_object(). ''' schema_type = schema.get('type') example = schema.get('example') - if example is not None: - return example - if schema_type == 'array' or (isinstance(schema_type, list) and 'array' in schema_type): + if borgmatic.config.schema.compare_types(schema_type, {'array'}): config = ruamel.yaml.comments.CommentedSeq( - [schema_to_sample_configuration(schema['items'], level, parent_is_sequence=True)] + example + if borgmatic.config.schema.compare_types( + schema['items'].get('type'), SCALAR_SCHEMA_TYPES + ) + else [ + schema_to_sample_configuration( + schema['items'], source_config, level, parent_is_sequence=True + ) + ] ) add_comments_to_configuration_sequence(config, schema, indent=(level * INDENT)) - elif schema_type == 'object' or (isinstance(schema_type, list) and 'object' in schema_type): - config = ruamel.yaml.comments.CommentedMap( - [ - (field_name, schema_to_sample_configuration(sub_schema, level + 1)) - for field_name, sub_schema in get_properties(schema).items() - ] + elif borgmatic.config.schema.compare_types(schema_type, {'object'}): + if source_config and isinstance(source_config, list) and isinstance(source_config[0], dict): + source_config = dict(collections.ChainMap(*source_config)) + + config = ( + ruamel.yaml.comments.CommentedMap( + [ + ( + field_name, + schema_to_sample_configuration( + sub_schema, (source_config or {}).get(field_name, {}), level + 1 + ), + ) + for field_name, sub_schema in borgmatic.config.schema.get_properties( + schema + ).items() + ] + ) + or example ) indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0) add_comments_to_configuration_object( - config, schema, indent=indent, skip_first=parent_is_sequence + config, schema, source_config, indent=indent, skip_first=parent_is_sequence ) + elif borgmatic.config.schema.compare_types(schema_type, SCALAR_SCHEMA_TYPES, match=all): + return example else: raise ValueError(f'Schema at level {level} is unsupported: {schema}') @@ -164,7 +179,7 @@ def add_comments_to_configuration_sequence(config, schema, indent=0): return for field_name in config[0].keys(): - field_schema = get_properties(schema['items']).get(field_name, {}) + field_schema = borgmatic.config.schema.get_properties(schema['items']).get(field_name, {}) description = field_schema.get('description') # No description to use? Skip it. @@ -178,26 +193,35 @@ def add_comments_to_configuration_sequence(config, schema, indent=0): return -REQUIRED_KEYS = {'source_directories', 'repositories', 'keep_daily'} +DEFAULT_KEYS = {'source_directories', 'repositories', 'keep_daily'} COMMENTED_OUT_SENTINEL = 'COMMENT_OUT' -def add_comments_to_configuration_object(config, schema, indent=0, skip_first=False): +def add_comments_to_configuration_object( + config, schema, source_config=None, indent=0, skip_first=False +): ''' Using descriptions from a schema as a source, add those descriptions as comments to the given - config mapping, before each field. Indent the comment the given number of characters. + configuration dict, putting them before each field. Indent the comment the given number of + characters. + + And a sentinel for commenting out options that are neither in DEFAULT_KEYS nor the the given + source configuration dict. The idea is that any options used in the source configuration should + stay active in the generated configuration. ''' for index, field_name in enumerate(config.keys()): if skip_first and index == 0: continue - field_schema = get_properties(schema).get(field_name, {}) + field_schema = borgmatic.config.schema.get_properties(schema).get(field_name, {}) description = field_schema.get('description', '').strip() - # If this is an optional key, add an indicator to the comment flagging it to be commented + # If this isn't a default key, add an indicator to the comment flagging it to be commented # out from the sample configuration. This sentinel is consumed by downstream processing that # does the actual commenting out. - if field_name not in REQUIRED_KEYS: + if field_name not in DEFAULT_KEYS and ( + source_config is None or field_name not in source_config + ): description = ( '\n'.join((description, COMMENTED_OUT_SENTINEL)) if description @@ -217,21 +241,6 @@ def add_comments_to_configuration_object(config, schema, indent=0, skip_first=Fa RUAMEL_YAML_COMMENTS_INDEX = 1 -def remove_commented_out_sentinel(config, field_name): - ''' - Given a configuration CommentedMap and a top-level field name in it, remove any "commented out" - sentinel found at the end of its YAML comments. This prevents the given field name from getting - commented out by downstream processing that consumes the sentinel. - ''' - try: - last_comment_value = config.ca.items[field_name][RUAMEL_YAML_COMMENTS_INDEX][-1].value - except KeyError: - return - - if last_comment_value == f'# {COMMENTED_OUT_SENTINEL}\n': - config.ca.items[field_name][RUAMEL_YAML_COMMENTS_INDEX].pop() - - def merge_source_configuration_into_destination(destination_config, source_config): ''' Deep merge the given source configuration dict into the destination configuration CommentedMap, @@ -246,12 +255,6 @@ def merge_source_configuration_into_destination(destination_config, source_confi return source_config for field_name, source_value in source_config.items(): - # Since this key/value is from the source configuration, leave it uncommented and remove any - # sentinel that would cause it to get commented out. - remove_commented_out_sentinel( - ruamel.yaml.comments.CommentedMap(destination_config), field_name - ) - # This is a mapping. Recurse for this key/value. if isinstance(source_value, collections.abc.Mapping): destination_config[field_name] = merge_source_configuration_into_destination( @@ -297,7 +300,7 @@ def generate_sample_configuration( normalize.normalize(source_filename, source_config) destination_config = merge_source_configuration_into_destination( - schema_to_sample_configuration(schema), source_config + schema_to_sample_configuration(schema, source_config), source_config ) if dry_run: diff --git a/borgmatic/config/load.py b/borgmatic/config/load.py index 978fc8e4..864bf3e5 100644 --- a/borgmatic/config/load.py +++ b/borgmatic/config/load.py @@ -69,7 +69,7 @@ def include_configuration(loader, filename_node, include_directory, config_paths ] raise ValueError( - '!include value is not supported; use a single filename or a list of filenames' + 'The value given for the !include tag is invalid; use a single filename or a list of filenames instead' ) diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index a869b24a..f4199e6b 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -58,6 +58,90 @@ def normalize_sections(config_filename, config): return [] +def make_command_hook_deprecation_log(config_filename, option_name): # pragma: no cover + ''' + Given a configuration filename and the name of a configuration option, return a deprecation + warning log for it. + ''' + return logging.makeLogRecord( + dict( + levelno=logging.WARNING, + levelname='WARNING', + msg=f'{config_filename}: {option_name} is deprecated and support will be removed from a future release. Use commands: instead.', + ) + ) + + +def normalize_commands(config_filename, config): + ''' + Given a configuration filename and a configuration dict, transform any "before_*"- and + "after_*"-style command hooks into "commands:". + ''' + logs = [] + + # Normalize "before_actions" and "after_actions". + for preposition in ('before', 'after'): + option_name = f'{preposition}_actions' + commands = config.pop(option_name, None) + + if commands: + logs.append(make_command_hook_deprecation_log(config_filename, option_name)) + config.setdefault('commands', []).append( + { + preposition: 'repository', + 'run': commands, + } + ) + + # Normalize "before_backup", "before_prune", "after_backup", "after_prune", etc. + for action_name in ('create', 'prune', 'compact', 'check', 'extract'): + for preposition in ('before', 'after'): + option_name = f'{preposition}_{"backup" if action_name == "create" else action_name}' + commands = config.pop(option_name, None) + + if not commands: + continue + + logs.append(make_command_hook_deprecation_log(config_filename, option_name)) + config.setdefault('commands', []).append( + { + preposition: 'action', + 'when': [action_name], + 'run': commands, + } + ) + + # Normalize "on_error". + commands = config.pop('on_error', None) + + if commands: + logs.append(make_command_hook_deprecation_log(config_filename, 'on_error')) + config.setdefault('commands', []).append( + { + 'after': 'error', + 'when': ['create', 'prune', 'compact', 'check'], + 'run': commands, + } + ) + + # Normalize "before_everything" and "after_everything". + for preposition in ('before', 'after'): + option_name = f'{preposition}_everything' + commands = config.pop(option_name, None) + + if commands: + logs.append(make_command_hook_deprecation_log(config_filename, option_name)) + config.setdefault('commands', []).append( + { + preposition: 'everything', + 'when': ['create'], + 'run': commands, + } + ) + + return logs + + def normalize(config_filename, config): ''' Given a configuration filename and a configuration dict of its loaded contents, apply particular @@ -67,6 +151,7 @@ def normalize(config_filename, config): Raise ValueError the configuration cannot be normalized. ''' logs = normalize_sections(config_filename, config) + logs += normalize_commands(config_filename, config) if config.get('borgmatic_source_directory'): logs.append( @@ -241,7 +326,11 @@ def normalize(config_filename, config): config['repositories'] = [] for repository_dict in repositories: - repository_path = repository_dict['path'] + repository_path = repository_dict.get('path') + + if repository_path is None: + continue + if '~' in repository_path: logs.append( logging.makeLogRecord( diff --git a/borgmatic/config/override.py b/borgmatic/config/override.py index 8a60cb50..9067e6f9 100644 --- a/borgmatic/config/override.py +++ b/borgmatic/config/override.py @@ -1,7 +1,10 @@ import io +import logging import ruamel.yaml +logger = logging.getLogger(__name__) + def set_values(config, keys, value): ''' @@ -134,6 +137,11 @@ def apply_overrides(config, schema, raw_overrides): ''' overrides = parse_overrides(raw_overrides, schema) + if overrides: + logger.warning( + "The --override flag is deprecated and will be removed from a future release. Instead, use a command-line flag corresponding to the configuration option you'd like to set." + ) + for keys, value in overrides: set_values(config, keys, value) set_values(config, strip_section_names(keys), value) diff --git a/borgmatic/config/paths.py b/borgmatic/config/paths.py index 7b13d306..48dff228 100644 --- a/borgmatic/config/paths.py +++ b/borgmatic/config/paths.py @@ -134,7 +134,7 @@ class Runtime_directory: ''' return self.runtime_path - def __exit__(self, exception, value, traceback): + def __exit__(self, exception_type, exception, traceback): ''' Delete any temporary directory that was created as part of initialization. ''' diff --git a/borgmatic/config/schema.py b/borgmatic/config/schema.py new file mode 100644 index 00000000..3d13c0c2 --- /dev/null +++ b/borgmatic/config/schema.py @@ -0,0 +1,72 @@ +import decimal +import itertools + + +def get_properties(schema): + ''' + Given a schema dict, return its properties. But if it's got sub-schemas with multiple different + potential properties, return their merged properties instead (interleaved so the first + properties of each sub-schema come first). The idea is that the user should see all possible + options even if they're not all possible together. + ''' + if 'oneOf' in schema: + return dict( + item + for item in itertools.chain( + *itertools.zip_longest( + *[sub_schema['properties'].items() for sub_schema in schema['oneOf']] + ) + ) + if item is not None + ) + + return schema.get('properties', {}) + + +SCHEMA_TYPE_TO_PYTHON_TYPE = { + 'array': list, + 'boolean': bool, + 'integer': int, + 'number': decimal.Decimal, + 'object': dict, + 'string': str, +} + + +def parse_type(schema_type, **overrides): + ''' + Given a schema type as a string, return the corresponding Python type. + + If any overrides are given in the from of a schema type string to a Python type, then override + the default type mapping with them. + + Raise ValueError if the schema type is unknown. + ''' + try: + return dict( + SCHEMA_TYPE_TO_PYTHON_TYPE, + **overrides, + )[schema_type] + except KeyError: + raise ValueError(f'Unknown type in configuration schema: {schema_type}') + + +def compare_types(schema_type, target_types, match=any): + ''' + Given a schema type as a string or a list of strings (representing multiple types) and a set of + target type strings, return whether every schema type is in the set of target types. + + If the schema type is a list of strings, use the given match function (such as any or all) to + compare elements. For instance, if match is given as all, then every element of the schema_type + list must be in the target types. + ''' + if isinstance(schema_type, list): + if match(element_schema_type in target_types for element_schema_type in schema_type): + return True + + return False + + if schema_type in target_types: + return True + + return False diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 0f647d1c..51b43567 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -33,13 +33,47 @@ properties: type: object required: - path + additionalProperties: false properties: path: type: string - example: ssh://user@backupserver/./{fqdn} + description: The local path or Borg URL of the repository. + example: ssh://user@backupserver/./sourcehostname.borg label: type: string + description: | + An optional label for the repository, used in logging + and to make selecting the repository easier on the + command-line. example: backupserver + encryption: + type: string + description: | + The encryption mode with which to create the repository, + only used for the repo-create action. To see the + available encryption modes, run "borg init --help" with + Borg 1 or "borg repo-create --help" with Borg 2. + example: repokey-blake2 + append_only: + type: boolean + description: | + Whether the repository should be created append-only, + only used for the repo-create action. Defaults to false. + example: true + storage_quota: + type: string + description: | + The storage quota with which to create the repository, + only used for the repo-create action. Defaults to no + quota. + example: 5G + make_parent_directories: + type: boolean + description: | + Whether any missing parent directories of the repository + path should be created, only used for the repo-create + action. Defaults to false. + example: true description: | A required list of local or remote repositories with paths and optional labels (which can be used with the --repository flag to @@ -48,8 +82,7 @@ properties: output of "borg help placeholders" for details. See ssh_command for SSH options like identity file or port. If systemd service is used, then add local repository paths in the systemd service file to the - ReadWritePaths list. Prior to borgmatic 1.7.10, repositories was a - list of plain path strings. + ReadWritePaths list. example: - path: ssh://user@backupserver/./sourcehostname.borg label: backupserver @@ -99,13 +132,13 @@ properties: used when backing up special devices such as /dev/zero. Defaults to false. But when a database hook is used, the setting here is ignored and read_special is considered true. - example: false + example: true flags: type: boolean description: | Record filesystem flags (e.g. NODUMP, IMMUTABLE) in archive. Defaults to true. - example: true + example: false files_cache: type: string description: | @@ -250,7 +283,7 @@ properties: repositories that were initialized with passphrase/repokey/keyfile encryption. Quote the value if it contains punctuation, so it parses correctly. And backslash any quote or backslash literals as well. - Defaults to not set. + Defaults to not set. Supports the "{credential ...}" syntax. example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" checkpoint_interval: type: integer @@ -284,6 +317,22 @@ properties: http://borgbackup.readthedocs.io/en/stable/usage/create.html for details. Defaults to "lz4". example: lz4 + recompress: + type: string + enum: ['if-different', 'always', 'never'] + description: | + Mode for recompressing data chunks according to MODE. + Possible modes are: + * "if-different": Recompress if the current compression + is with a different compression algorithm. + * "always": Recompress even if the current compression + is with the same compression algorithm. Use this to change + the compression level. + * "never": Do not recompress. Use this option to explicitly + prevent recompression. + See https://borgbackup.readthedocs.io/en/stable/usage/recreate.html + for details. Defaults to "never". + example: if-different upload_rate_limit: type: integer description: | @@ -426,19 +475,19 @@ properties: type: boolean description: | Bypass Borg error about a repository that has been moved. Defaults - to not bypassing. + to false. example: true unknown_unencrypted_repo_access_is_ok: type: boolean description: | Bypass Borg error about a previously unknown unencrypted repository. - Defaults to not bypassing. + Defaults to false. example: true check_i_know_what_i_am_doing: type: boolean description: | Bypass Borg confirmation about check with repair option. Defaults to - an interactive prompt from Borg. + false and an interactive prompt from Borg. example: true extra_borg_options: type: object @@ -518,6 +567,12 @@ properties: not specified, borgmatic defaults to matching archives based on the archive_name_format (see above). example: sourcehostname + compact_threshold: + type: integer + description: | + Minimum saved space percentage threshold for compacting a segment, + defaults to 10. + example: 20 checks: type: array items: @@ -733,6 +788,10 @@ properties: List of one or more consistency checks to run on a periodic basis (if "frequency" is set) or every time borgmatic runs checks (if "frequency" is omitted). + example: + - name: archives + frequency: 2 weeks + - name: repository check_repositories: type: array items: @@ -855,8 +914,31 @@ properties: false. example: true description: | - Settings for where to log borgmatic progress and how verbose those + Settings for where to log borgmatic output and how verbose those logs should be. + example: false + progress: + type: boolean + description: | + Display progress as each file or archive is processed when running + supported actions. Corresponds to the "--progress" flag on those + actions. Defaults to false. + example: true + statistics: + type: boolean + description: | + Display statistics for an archive when running supported actions. + Corresponds to the "--stats" flag on those actions. Defaults to + false. + example: true + list_details: + type: boolean + description: | + Display details for each file or archive as it is processed when + running supported actions. Corresponds to the "--list" flag on those + actions. Defaults to false. + example: true +>>>>>>> main skip_actions: type: array items: @@ -867,6 +949,7 @@ properties: - prune - compact - create + - recreate - check - delete - extract @@ -896,8 +979,9 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute before all - the actions for each repository. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute before all the actions for each + repository. example: - "echo Starting actions." before_backup: @@ -905,8 +989,9 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute before - creating a backup, run once per repository. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute before creating a backup, run once + per repository. example: - "echo Starting a backup." before_prune: @@ -914,8 +999,9 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute before - pruning, run once per repository. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute before pruning, run once per + repository. example: - "echo Starting pruning." before_compact: @@ -923,8 +1009,9 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute before - compaction, run once per repository. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute before compaction, run once per + repository. example: - "echo Starting compaction." before_check: @@ -932,8 +1019,9 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute before - consistency checks, run once per repository. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute before consistency checks, run once + per repository. example: - "echo Starting checks." before_extract: @@ -941,8 +1029,9 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute before - extracting a backup, run once per repository. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute before extracting a backup, run once + per repository. example: - "echo Starting extracting." after_backup: @@ -950,8 +1039,9 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute after - creating a backup, run once per repository. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute after creating a backup, run once per + repository. example: - "echo Finished a backup." after_compact: @@ -959,8 +1049,9 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute after - compaction, run once per repository. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute after compaction, run once per + repository. example: - "echo Finished compaction." after_prune: @@ -968,8 +1059,9 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute after - pruning, run once per repository. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute after pruning, run once per + repository. example: - "echo Finished pruning." after_check: @@ -977,8 +1069,9 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute after - consistency checks, run once per repository. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute after consistency checks, run once + per repository. example: - "echo Finished checks." after_extract: @@ -986,8 +1079,9 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute after - extracting a backup, run once per repository. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute after extracting a backup, run once + per repository. example: - "echo Finished extracting." after_actions: @@ -995,8 +1089,9 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute after all - actions for each repository. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute after all actions for each + repository. example: - "echo Finished actions." on_error: @@ -1004,9 +1099,10 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute when an - exception occurs during a "create", "prune", "compact", or "check" - action or an associated before/after hook. + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute when an exception occurs during a + "create", "prune", "compact", or "check" action or an associated + before/after hook. example: - "echo Error during create/prune/compact/check." before_everything: @@ -1014,10 +1110,10 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute before - running all actions (if one of them is "create"). These are - collected from all configuration files and then run once before all - of them (prior to all actions). + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute before running all actions (if one of + them is "create"). These are collected from all configuration files + and then run once before all of them (prior to all actions). example: - "echo Starting actions." after_everything: @@ -1025,14 +1121,157 @@ properties: items: type: string description: | - List of one or more shell commands or scripts to execute after - running all actions (if one of them is "create"). These are - collected from all configuration files and then run once after all - of them (after any action). + Deprecated. Use "commands:" instead. List of one or more shell + commands or scripts to execute after running all actions (if one of + them is "create"). These are collected from all configuration files + and then run once after all of them (after any action). example: - "echo Completed actions." + commands: + type: array + items: + type: object + oneOf: + - required: [before, run] + additionalProperties: false + properties: + before: + type: string + enum: + - action + - repository + - configuration + - everything + description: | + Name for the point in borgmatic's execution that + the commands should be run before (required if + "after" isn't set): + * "action" runs before each action for each + repository. + * "repository" runs before all actions for each + repository. + * "configuration" runs before all actions and + repositories in the current configuration file. + * "everything" runs before all configuration + files. + example: action + when: + type: array + items: + type: string + enum: + - repo-create + - transfer + - prune + - compact + - create + - recreate + - check + - delete + - extract + - config + - export-tar + - mount + - umount + - repo-delete + - restore + - repo-list + - list + - repo-info + - info + - break-lock + - key + - borg + description: | + List of actions for which the commands will be + run. Defaults to running for all actions. + example: [create, prune, compact, check] + run: + type: array + items: + type: string + description: | + List of one or more shell commands or scripts to + run when this command hook is triggered. Required. + example: + - "echo Doing stuff." + - required: [after, run] + additionalProperties: false + properties: + after: + type: string + enum: + - action + - repository + - configuration + - everything + - error + description: | + Name for the point in borgmatic's execution that + the commands should be run after (required if + "before" isn't set): + * "action" runs after each action for each + repository. + * "repository" runs after all actions for each + repository. + * "configuration" runs after all actions and + repositories in the current configuration file. + * "everything" runs after all configuration + files. + * "error" runs after an error occurs. + example: action + when: + type: array + items: + type: string + enum: + - repo-create + - transfer + - prune + - compact + - create + - recreate + - check + - delete + - extract + - config + - export-tar + - mount + - umount + - repo-delete + - restore + - repo-list + - list + - repo-info + - info + - break-lock + - key + - borg + description: | + Only trigger the hook when borgmatic is run with + particular actions listed here. Defaults to + running for all actions. + example: [create, prune, compact, check] + run: + type: array + items: + type: string + description: | + List of one or more shell commands or scripts to + run when this command hook is triggered. Required. + example: + - "echo Doing stuff." + description: | + List of one or more command hooks to execute, triggered at + particular points during borgmatic's execution. For each command + hook, specify one of "before" or "after", not both. + example: + - before: action + when: [create] + run: [echo Backing up.] bootstrap: type: object + additionalProperties: false properties: store_config_files: type: boolean @@ -1089,13 +1328,15 @@ properties: Username with which to connect to the database. Defaults to the username of the current user. You probably want to specify the "postgres" superuser here when the - database name is "all". + database name is "all". Supports the "{credential ...}" + syntax. example: dbuser restore_username: type: string description: | Username with which to restore the database. Defaults to - the "username" option. + the "username" option. Supports the "{credential ...}" + syntax. example: dbuser password: type: string @@ -1103,13 +1344,15 @@ properties: Password with which to connect to the database. Omitting a password will only work if PostgreSQL is configured to trust the configured username without a password or you - create a ~/.pgpass file. + create a ~/.pgpass file. Supports the "{credential ...}" + syntax. example: trustsome1 restore_password: type: string description: | Password with which to connect to the restore database. - Defaults to the "password" option. + Defaults to the "password" option. Supports the + "{credential ...}" syntax. example: trustsome1 no_owner: type: boolean @@ -1136,6 +1379,18 @@ properties: individual databases. See the pg_dump documentation for more about formats. example: directory + compression: + type: ["string", "integer"] + description: | + Database dump compression level (integer) or method + ("gzip", "lz4", "zstd", or "none") and optional + colon-separated detail. Defaults to moderate "gzip" for + "custom" and "directory" formats and no compression for + the "plain" format. Compression is not supported for the + "tar" format. Be aware that Borg does its own + compression as well, so you may not need it in both + places. + example: none ssl_mode: type: string enum: ['disable', 'allow', 'prefer', @@ -1172,11 +1427,11 @@ properties: Command to use instead of "pg_dump" or "pg_dumpall". This can be used to run a specific pg_dump version (e.g., one inside a running container). If you run it - from within a container, make sure to mount your - host's ".borgmatic" folder into the container using - the same directory structure. Defaults to "pg_dump" - for single database dump or "pg_dumpall" to dump all - databases. + from within a container, make sure to mount the path in + the "user_runtime_directory" option from the host into + the container at the same location. Defaults to + "pg_dump" for single database dump or "pg_dumpall" to + dump all databases. example: docker exec my_pg_container pg_dump pg_restore_command: type: string @@ -1229,6 +1484,9 @@ properties: https://www.postgresql.org/docs/current/app-pgdump.html and https://www.postgresql.org/docs/current/libpq-ssl.html for details. + example: + - name: users + hostname: database.example.org mariadb_databases: type: array items: @@ -1269,13 +1527,15 @@ properties: type: string description: | Username with which to connect to the database. Defaults - to the username of the current user. + to the username of the current user. Supports the + "{credential ...}" syntax. example: dbuser restore_username: type: string description: | Username with which to restore the database. Defaults to - the "username" option. + the "username" option. Supports the "{credential ...}" + syntax. example: dbuser password: type: string @@ -1283,16 +1543,39 @@ properties: Password with which to connect to the database. Omitting a password will only work if MariaDB is configured to trust the configured username without a password. + Supports the "{credential ...}" syntax. example: trustsome1 + restore_password: + type: string + description: | + Password with which to connect to the restore database. + Defaults to the "password" option. Supports the + "{credential ...}" syntax. + example: trustsome1 + tls: + type: boolean + description: | + Whether to TLS-encrypt data transmitted between the + client and server. The default varies based on the + MariaDB version. + example: false + restore_tls: + type: boolean + description: | + Whether to TLS-encrypt data transmitted between the + client and restore server. The default varies based on + the MariaDB version. + example: false mariadb_dump_command: type: string description: | Command to use instead of "mariadb-dump". This can be used to run a specific mariadb_dump version (e.g., one - inside a running container). If you run it from within - a container, make sure to mount your host's - ".borgmatic" folder into the container using the same - directory structure. Defaults to "mariadb-dump". + inside a running container). If you run it from within a + container, make sure to mount the path in the + "user_runtime_directory" option from the host into the + container at the same location. Defaults to + "mariadb-dump". example: docker exec mariadb_container mariadb-dump mariadb_command: type: string @@ -1301,12 +1584,6 @@ properties: run a specific mariadb version (e.g., one inside a running container). Defaults to "mariadb". example: docker exec mariadb_container mariadb - restore_password: - type: string - description: | - Password with which to connect to the restore database. - Defaults to the "password" option. - example: trustsome1 format: type: string enum: ['sql'] @@ -1355,6 +1632,9 @@ properties: added to your source directories at runtime and streamed directly to Borg. Requires mariadb-dump/mariadb commands. See https://mariadb.com/kb/en/library/mysqldump/ for details. + example: + - name: users + hostname: database.example.org mysql_databases: type: array items: @@ -1395,13 +1675,15 @@ properties: type: string description: | Username with which to connect to the database. Defaults - to the username of the current user. + to the username of the current user. Supports the + "{credential ...}" syntax. example: dbuser restore_username: type: string description: | Username with which to restore the database. Defaults to - the "username" option. + the "username" option. Supports the "{credential ...}" + syntax. example: dbuser password: type: string @@ -1409,22 +1691,38 @@ properties: Password with which to connect to the database. Omitting a password will only work if MySQL is configured to trust the configured username without a password. + Supports the "{credential ...}" syntax. example: trustsome1 restore_password: type: string description: | Password with which to connect to the restore database. - Defaults to the "password" option. + Defaults to the "password" option. Supports the + "{credential ...}" syntax. example: trustsome1 + tls: + type: boolean + description: | + Whether to TLS-encrypt data transmitted between the + client and server. The default varies based on the + MySQL installation. + example: false + restore_tls: + type: boolean + description: | + Whether to TLS-encrypt data transmitted between the + client and restore server. The default varies based on + the MySQL installation. + example: false mysql_dump_command: type: string description: | - Command to use instead of "mysqldump". This can be - used to run a specific mysql_dump version (e.g., one - inside a running container). If you run it from within - a container, make sure to mount your host's - ".borgmatic" folder into the container using the same - directory structure. Defaults to "mysqldump". + Command to use instead of "mysqldump". This can be used + to run a specific mysql_dump version (e.g., one inside a + running container). If you run it from within a + container, make sure to mount the path in the + "user_runtime_directory" option from the host into the + container at the same location. Defaults to "mysqldump". example: docker exec mysql_container mysqldump mysql_command: type: string @@ -1482,6 +1780,9 @@ properties: to Borg. Requires mysqldump/mysql commands. See https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html for details. + example: + - name: users + hostname: database.example.org sqlite_databases: type: array items: @@ -1511,6 +1812,33 @@ properties: Path to the SQLite database file to restore to. Defaults to the "path" option. example: /var/lib/sqlite/users.db + sqlite_command: + type: string + description: | + Command to use instead of "sqlite3". This can be used to + run a specific sqlite3 version (e.g., one inside a + running container). If you run it from within a + container, make sure to mount the path in the + "user_runtime_directory" option from the host into the + container at the same location. Defaults to "sqlite3". + example: docker exec sqlite_container sqlite3 + sqlite_restore_command: + type: string + description: | + Command to run when restoring a database instead + of "sqlite3". This can be used to run a specific + sqlite3 version (e.g., one inside a running container). + Defaults to "sqlite3". + example: docker exec sqlite_container sqlite3 + description: | + List of one or more SQLite databases to dump before creating a + backup, run once per configuration file. The database dumps are + added to your source directories at runtime and streamed directly to + Borg. Requires the sqlite3 command. See https://sqlite.org/cli.html + for details. + example: + - name: users + path: /var/lib/db.sqlite mongodb_databases: type: array items: @@ -1551,25 +1879,29 @@ properties: type: string description: | Username with which to connect to the database. Skip it - if no authentication is needed. + if no authentication is needed. Supports the + "{credential ...}" syntax. example: dbuser restore_username: type: string description: | Username with which to restore the database. Defaults to - the "username" option. + the "username" option. Supports the "{credential ...}" + syntax. example: dbuser password: type: string description: | Password with which to connect to the database. Skip it - if no authentication is needed. + if no authentication is needed. Supports the + "{credential ...}" syntax. example: trustsome1 restore_password: type: string description: | Password with which to connect to the restore database. - Defaults to the "password" option. + Defaults to the "password" option. Supports the + "{credential ...}" syntax. example: trustsome1 authentication_database: type: string @@ -1602,6 +1934,25 @@ properties: dump command, without performing any validation on them. See mongorestore documentation for details. example: --restoreDbUsersAndRoles + mongodump_command: + type: string + description: | + Command to use instead of "mongodump". This can be used + to run a specific mongodump version (e.g., one inside a + running container). If you run it from within a + container, make sure to mount the path in the + "user_runtime_directory" option from the host into the + container at the same location. Defaults to + "mongodump". + example: docker exec mongodb_container mongodump + mongorestore_command: + type: string + description: | + Command to run when restoring a database instead of + "mongorestore". This can be used to run a specific + mongorestore version (e.g., one inside a running + container). Defaults to "mongorestore". + example: docker exec mongodb_container mongorestore description: | List of one or more MongoDB databases to dump before creating a backup, run once per configuration file. The database dumps are @@ -1609,6 +1960,9 @@ properties: to Borg. Requires mongodump/mongorestore commands. See https://docs.mongodb.com/database-tools/mongodump/ and https://docs.mongodb.com/database-tools/mongorestore/ for details. + example: + - name: users + hostname: database.example.org ntfy: type: object required: ['topic'] @@ -1628,21 +1982,24 @@ properties: username: type: string description: | - The username used for authentication. + The username used for authentication. Supports the + "{credential ...}" syntax. example: testuser password: type: string description: | - The password used for authentication. + The password used for authentication. Supports the + "{credential ...}" syntax. example: fakepassword access_token: type: string description: | An ntfy access token to authenticate with instead of - username/password. + username/password. Supports the "{credential ...}" syntax. example: tk_AgQdq7mVBoFD37zQVN29RhuMzNIz2 start: type: object + additionalProperties: false properties: title: type: string @@ -1666,6 +2023,7 @@ properties: example: incoming_envelope finish: type: object + additionalProperties: false properties: title: type: string @@ -1689,6 +2047,7 @@ properties: example: incoming_envelope fail: type: object + additionalProperties: false properties: title: type: string @@ -1734,17 +2093,20 @@ properties: token: type: string description: | - Your application's API token. + Your application's API token. Supports the "{credential + ...}" syntax. example: 7ms6TXHpTokTou2P6x4SodDeentHRa user: type: string description: | - Your user/group key (or that of your target user), viewable - when logged into your dashboard: often referred to as + Your user/group key (or that of your target user), viewable + when logged into your dashboard: often referred to as USER_KEY in Pushover documentation and code examples. + Supports the "{credential ...}" syntax. example: hwRwoWsXMBWwgrSecfa9EfPey55WSN start: type: object + additionalProperties: false properties: message: type: string @@ -1784,8 +2146,8 @@ properties: type: boolean description: | Set to True to enable HTML parsing of the message. - Set to False for plain text. - example: True + Set to false for plain text. + example: true sound: type: string description: | @@ -1820,6 +2182,7 @@ properties: example: Pushover Link finish: type: object + additionalProperties: false properties: message: type: string @@ -1859,8 +2222,8 @@ properties: type: boolean description: | Set to True to enable HTML parsing of the message. - Set to False for plain text. - example: True + Set to false for plain text. + example: true sound: type: string description: | @@ -1895,6 +2258,7 @@ properties: example: Pushover Link fail: type: object + additionalProperties: false properties: message: type: string @@ -1934,8 +2298,8 @@ properties: type: boolean description: | Set to True to enable HTML parsing of the message. - Set to False for plain text. - example: True + Set to false for plain text. + example: true sound: type: string description: | @@ -1987,6 +2351,8 @@ properties: zabbix: type: object additionalProperties: false + required: + - server properties: itemid: type: integer @@ -2009,28 +2375,30 @@ properties: server: type: string description: | - The address of your Zabbix instance. + The API endpoint URL of your Zabbix instance, usually ending + with "/api_jsonrpc.php". Required. example: https://zabbix.your-domain.com username: type: string description: | The username used for authentication. Not needed if using - an API key. + an API key. Supports the "{credential ...}" syntax. example: testuser password: type: string description: | The password used for authentication. Not needed if using - an API key. + an API key. Supports the "{credential ...}" syntax. example: fakepassword api_key: type: string description: | - The API key used for authentication. Not needed if using - an username/password. + The API key used for authentication. Not needed if using an + username/password. Supports the "{credential ...}" syntax. example: fakekey start: type: object + additionalProperties: false properties: value: type: ["integer", "string"] @@ -2039,6 +2407,7 @@ properties: example: STARTED finish: type: object + additionalProperties: false properties: value: type: ["integer", "string"] @@ -2047,6 +2416,7 @@ properties: example: FINISH fail: type: object + additionalProperties: false properties: value: type: ["integer", "string"] @@ -2078,15 +2448,20 @@ properties: type: array items: type: object + additionalProperties: false required: - url - label properties: url: type: string + description: URL of this Apprise service. example: "gotify://hostname/token" label: type: string + description: | + Label used in borgmatic logs for this Apprise + service. example: gotify description: | A list of Apprise services to publish to with URLs and @@ -2101,7 +2476,7 @@ properties: send_logs: type: boolean description: | - Send borgmatic logs to Apprise services as part the + Send borgmatic logs to Apprise services as part of the "finish", "fail", and "log" states. Defaults to true. example: false logs_size_limit: @@ -2114,6 +2489,7 @@ properties: start: type: object required: ['body'] + additionalProperties: false properties: title: type: string @@ -2129,6 +2505,7 @@ properties: finish: type: object required: ['body'] + additionalProperties: false properties: title: type: string @@ -2144,6 +2521,7 @@ properties: fail: type: object required: ['body'] + additionalProperties: false properties: title: type: string @@ -2159,6 +2537,7 @@ properties: log: type: object required: ['body'] + additionalProperties: false properties: title: type: string @@ -2212,7 +2591,7 @@ properties: send_logs: type: boolean description: | - Send borgmatic logs to Healthchecks as part the "finish", + Send borgmatic logs to Healthchecks as part of the "finish", "fail", and "log" states. Defaults to true. example: false ping_body_limit: @@ -2280,6 +2659,12 @@ properties: - start - finish - fail + verify_tls: + type: boolean + description: | + Verify the TLS certificate of the push URL host. Defaults to + true. + example: false description: | Configuration for a monitoring integration with Uptime Kuma using the Push monitor type. @@ -2307,9 +2692,15 @@ properties: integration_key: type: string description: | - PagerDuty integration key used to notify PagerDuty - when a backup errors. + PagerDuty integration key used to notify PagerDuty when a + backup errors. Supports the "{credential ...}" syntax. example: a177cad45bd374409f78906a810a3074 + send_logs: + type: boolean + description: | + Send borgmatic logs to PagerDuty when a backup errors. + Defaults to true. + example: false description: | Configuration for a monitoring integration with PagerDuty. Create an account at https://www.pagerduty.com if you'd like to use this @@ -2482,3 +2873,47 @@ properties: description: | Configuration for integration with Linux LVM (Logical Volume Manager). + container: + type: object + additionalProperties: false + properties: + secrets_directory: + type: string + description: | + Secrets directory to use instead of "/run/secrets". + example: /path/to/secrets + description: | + Configuration for integration with Docker or Podman secrets. + keepassxc: + type: object + additionalProperties: false + properties: + keepassxc_cli_command: + type: string + description: | + Command to use instead of "keepassxc-cli". + example: /usr/local/bin/keepassxc-cli + key_file: + type: string + description: | + Path to a key file for unlocking the KeePassXC database. + example: /path/to/keyfile + yubikey: + type: string + description: | + YubiKey slot and optional serial number used to access the + KeePassXC database. The format is "", where: + * is the YubiKey slot number (e.g., `1` or `2`). + * (optional) is the YubiKey's serial number (e.g., + `7370001`). + example: "1:7370001" + description: | + Configuration for integration with the KeePassXC password manager. + default_actions: + type: boolean + description: | + Whether to apply default actions (e.g., backup) when no arguments + are supplied to the borgmatic command. If set to true, borgmatic + triggers the default actions (create, prune, compact and check). If + set to false, borgmatic displays the help message instead. + example: true diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index dce99635..9cb02243 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -4,7 +4,7 @@ import os import jsonschema import ruamel.yaml -import borgmatic.config +import borgmatic.config.arguments from borgmatic.config import constants, environment, load, normalize, override @@ -21,6 +21,18 @@ def schema_filename(): return schema_path +def load_schema(schema_path): # pragma: no cover + ''' + Given a schema filename path, load the schema and return it as a dict. + + Raise Validation_error if the schema could not be parsed. + ''' + try: + return load.load_configuration(schema_path) + except (ruamel.yaml.error.YAMLError, RecursionError) as error: + raise Validation_error(schema_path, (str(error),)) + + def format_json_error_path_element(path_element): ''' Given a path element into a JSON data structure, format it for display as a string. @@ -84,12 +96,17 @@ def apply_logical_validation(config_filename, parsed_configuration): ) -def parse_configuration(config_filename, schema_filename, overrides=None, resolve_env=True): +def parse_configuration( + config_filename, schema_filename, arguments, overrides=None, resolve_env=True +): ''' Given the path to a config filename in YAML format, the path to a schema filename in a YAML - rendition of JSON Schema format, a sequence of configuration file override strings in the form - of "option.suboption=value", return the parsed configuration as a data structure of nested dicts - and lists corresponding to the schema. Example return value: + rendition of JSON Schema format, arguments as dict from action name to argparse.Namespace, a + sequence of configuration file override strings in the form of "option.suboption=value", and + whether to resolve environment variables, return the parsed configuration as a data structure of + nested dicts and lists corresponding to the schema. Example return value. + + Example return value: { 'source_directories': ['/home', '/etc'], @@ -112,6 +129,7 @@ def parse_configuration(config_filename, schema_filename, overrides=None, resolv except (ruamel.yaml.error.YAMLError, RecursionError) as error: raise Validation_error(config_filename, (str(error),)) + borgmatic.config.arguments.apply_arguments_to_config(config, schema, arguments) override.apply_overrides(config, schema, overrides) constants.apply_constants(config, config.get('constants') if config else {}) @@ -124,6 +142,7 @@ def parse_configuration(config_filename, schema_filename, overrides=None, resolv validator = jsonschema.Draft7Validator(schema) except AttributeError: # pragma: no cover validator = jsonschema.Draft4Validator(schema) + validation_errors = tuple(validator.iter_errors(config)) if validation_errors: @@ -136,16 +155,22 @@ def parse_configuration(config_filename, schema_filename, overrides=None, resolv return config, config_paths, logs -def normalize_repository_path(repository): +def normalize_repository_path(repository, base=None): ''' Given a repository path, return the absolute path of it (for local repositories). + Optionally, use a base path for resolving relative paths, e.g. to the configured working directory. ''' # A colon in the repository could mean that it's either a file:// URL or a remote repository. # If it's a remote repository, we don't want to normalize it. If it's a file:// URL, we do. if ':' not in repository: - return os.path.abspath(repository) + return ( + os.path.abspath(os.path.join(base, repository)) if base else os.path.abspath(repository) + ) elif repository.startswith('file://'): - return os.path.abspath(repository.partition('file://')[-1]) + local_path = repository.partition('file://')[-1] + return ( + os.path.abspath(os.path.join(base, local_path)) if base else os.path.abspath(local_path) + ) else: return repository diff --git a/borgmatic/execute.py b/borgmatic/execute.py index 8d850496..20eedb58 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -1,7 +1,6 @@ import collections import enum import logging -import os import select import subprocess import textwrap @@ -243,6 +242,9 @@ def mask_command_secrets(full_command): MAX_LOGGED_COMMAND_LENGTH = 1000 +PREFIXES_OF_ENVIRONMENT_VARIABLES_TO_LOG = ('BORG_', 'PG', 'MARIADB_', 'MYSQL_') + + def log_command(full_command, input_file=None, output_file=None, environment=None): ''' Log the given command (a sequence of command/argument strings), along with its input/output file @@ -251,14 +253,21 @@ def log_command(full_command, input_file=None, output_file=None, environment=Non logger.debug( textwrap.shorten( ' '.join( - tuple(f'{key}=***' for key in (environment or {}).keys()) + tuple( + f'{key}=***' + for key in (environment or {}).keys() + if any( + key.startswith(prefix) + for prefix in PREFIXES_OF_ENVIRONMENT_VARIABLES_TO_LOG + ) + ) + mask_command_secrets(full_command) ), width=MAX_LOGGED_COMMAND_LENGTH, placeholder=' ...', ) - + (f" < {getattr(input_file, 'name', '')}" if input_file else '') - + (f" > {getattr(output_file, 'name', '')}" if output_file else '') + + (f" < {getattr(input_file, 'name', input_file)}" if input_file else '') + + (f" > {getattr(output_file, 'name', output_file)}" if output_file else '') ) @@ -274,7 +283,7 @@ def execute_command( output_file=None, input_file=None, shell=False, - extra_environment=None, + environment=None, working_directory=None, borg_local_path=None, borg_exit_codes=None, @@ -284,18 +293,17 @@ def execute_command( Execute the given command (a sequence of command/argument strings) and log its output at the given log level. If an open output file object is given, then write stdout to the file and only log stderr. If an open input file object is given, then read stdin from the file. If shell is - True, execute the command within a shell. If an extra environment dict is given, then use it to - augment the current environment, and pass the result into the command. If a working directory is - given, use that as the present working directory when running the command. If a Borg local path - is given, and the command matches it (regardless of arguments), treat exit code 1 as a warning - instead of an error. But if Borg exit codes are given as a sequence of exit code configuration - dicts, then use that configuration to decide what's an error and what's a warning. If run to - completion is False, then return the process for the command without executing it to completion. + True, execute the command within a shell. If an environment variables dict is given, then pass + it into the command. If a working directory is given, use that as the present working directory + when running the command. If a Borg local path is given, and the command matches it (regardless + of arguments), treat exit code 1 as a warning instead of an error. But if Borg exit codes are + given as a sequence of exit code configuration dicts, then use that configuration to decide + what's an error and what's a warning. If run to completion is False, then return the process for + the command without executing it to completion. Raise subprocesses.CalledProcessError if an error occurs while running the command. ''' - log_command(full_command, input_file, output_file, extra_environment) - environment = {**os.environ, **extra_environment} if extra_environment else None + log_command(full_command, input_file, output_file, environment) do_not_capture = bool(output_file is DO_NOT_CAPTURE) command = ' '.join(full_command) if shell else full_command @@ -307,8 +315,8 @@ def execute_command( shell=shell, env=environment, cwd=working_directory, - # Necessary for the passcommand credential hook to work. - close_fds=not bool((extra_environment or {}).get('BORG_PASSPHRASE_FD')), + # Necessary for passing credentials via anonymous pipe. + close_fds=False, ) if not run_to_completion: return process @@ -325,39 +333,40 @@ def execute_command( def execute_command_and_capture_output( full_command, + input_file=None, capture_stderr=False, shell=False, - extra_environment=None, + environment=None, working_directory=None, borg_local_path=None, borg_exit_codes=None, ): ''' Execute the given command (a sequence of command/argument strings), capturing and returning its - output (stdout). If capture stderr is True, then capture and return stderr in addition to - stdout. If shell is True, execute the command within a shell. If an extra environment dict is - given, then use it to augment the current environment, and pass the result into the command. If - a working directory is given, use that as the present working directory when running the - command. If a Borg local path is given, and the command matches it (regardless of arguments), - treat exit code 1 as a warning instead of an error. But if Borg exit codes are given as a - sequence of exit code configuration dicts, then use that configuration to decide what's an error - and what's a warning. + output (stdout). If an input file descriptor is given, then pipe it to the command's stdin. If + capture stderr is True, then capture and return stderr in addition to stdout. If shell is True, + execute the command within a shell. If an environment variables dict is given, then pass it into + the command. If a working directory is given, use that as the present working directory when + running the command. If a Borg local path is given, and the command matches it (regardless of + arguments), treat exit code 1 as a warning instead of an error. But if Borg exit codes are given + as a sequence of exit code configuration dicts, then use that configuration to decide what's an + error and what's a warning. Raise subprocesses.CalledProcessError if an error occurs while running the command. ''' - log_command(full_command, environment=extra_environment) - environment = {**os.environ, **extra_environment} if extra_environment else None + log_command(full_command, input_file, environment=environment) command = ' '.join(full_command) if shell else full_command try: output = subprocess.check_output( command, + stdin=input_file, stderr=subprocess.STDOUT if capture_stderr else None, shell=shell, env=environment, cwd=working_directory, - # Necessary for the passcommand credential hook to work. - close_fds=not bool((extra_environment or {}).get('BORG_PASSPHRASE_FD')), + # Necessary for passing credentials via anonymous pipe. + close_fds=False, ) except subprocess.CalledProcessError as error: if ( @@ -377,7 +386,7 @@ def execute_command_with_processes( output_file=None, input_file=None, shell=False, - extra_environment=None, + environment=None, working_directory=None, borg_local_path=None, borg_exit_codes=None, @@ -391,19 +400,17 @@ def execute_command_with_processes( If an open output file object is given, then write stdout to the file and only log stderr. But if output log level is None, instead suppress logging and return the captured output for (only) the given command. If an open input file object is given, then read stdin from the file. If - shell is True, execute the command within a shell. If an extra environment dict is given, then - use it to augment the current environment, and pass the result into the command. If a working - directory is given, use that as the present working directory when running the command. If a - Borg local path is given, then for any matching command or process (regardless of arguments), - treat exit code 1 as a warning instead of an error. But if Borg exit codes are given as a - sequence of exit code configuration dicts, then use that configuration to decide what's an error - and what's a warning. + shell is True, execute the command within a shell. If an environment variables dict is given, + then pass it into the command. If a working directory is given, use that as the present working + directory when running the command. If a Borg local path is given, then for any matching command + or process (regardless of arguments), treat exit code 1 as a warning instead of an error. But if + Borg exit codes are given as a sequence of exit code configuration dicts, then use that + configuration to decide what's an error and what's a warning. Raise subprocesses.CalledProcessError if an error occurs while running the command or in the upstream process. ''' - log_command(full_command, input_file, output_file, extra_environment) - environment = {**os.environ, **extra_environment} if extra_environment else None + log_command(full_command, input_file, output_file, environment) do_not_capture = bool(output_file is DO_NOT_CAPTURE) command = ' '.join(full_command) if shell else full_command @@ -418,8 +425,8 @@ def execute_command_with_processes( shell=shell, env=environment, cwd=working_directory, - # Necessary for the passcommand credential hook to work. - close_fds=not bool((extra_environment or {}).get('BORG_PASSPHRASE_FD')), + # Necessary for passing credentials via anonymous pipe. + close_fds=False, ) except (subprocess.CalledProcessError, OSError): # Something has gone wrong. So vent each process' output buffer to prevent it from hanging. @@ -430,13 +437,14 @@ def execute_command_with_processes( process.kill() raise - captured_outputs = log_outputs( - tuple(processes) + (command_process,), - (input_file, output_file), - output_log_level, - borg_local_path, - borg_exit_codes, - ) + with borgmatic.logger.Log_prefix(None): # Log command output without any prefix. + captured_outputs = log_outputs( + tuple(processes) + (command_process,), + (input_file, output_file), + output_log_level, + borg_local_path, + borg_exit_codes, + ) if output_log_level is None: return captured_outputs.get(command_process) diff --git a/borgmatic/hooks/command.py b/borgmatic/hooks/command.py index d446ecca..eff1a752 100644 --- a/borgmatic/hooks/command.py +++ b/borgmatic/hooks/command.py @@ -2,9 +2,11 @@ import logging import os import re import shlex +import subprocess import sys import borgmatic.execute +import borgmatic.logger logger = logging.getLogger(__name__) @@ -30,66 +32,198 @@ def interpolate_context(hook_description, command, context): def make_environment(current_environment, sys_module=sys): ''' - Given the existing system environment as a map from environment variable name to value, return - (in the same form) any extra environment variables that should be used when running command - hooks. + Given the existing system environment as a map from environment variable name to value, return a + copy of it, augmented with any extra environment variables that should be used when running + command hooks. ''' + environment = dict(current_environment) + # Detect whether we're running within a PyInstaller bundle. If so, set or clear LD_LIBRARY_PATH # based on the value of LD_LIBRARY_PATH_ORIG. This prevents library version information errors. if getattr(sys_module, 'frozen', False) and hasattr(sys_module, '_MEIPASS'): - return {'LD_LIBRARY_PATH': current_environment.get('LD_LIBRARY_PATH_ORIG', '')} + environment['LD_LIBRARY_PATH'] = environment.get('LD_LIBRARY_PATH_ORIG', '') - return {} + return environment -def execute_hook(commands, umask, config_filename, description, dry_run, **context): +def filter_hooks(command_hooks, before=None, after=None, hook_name=None, action_names=None): ''' - Given a list of hook commands to execute, a umask to execute with (or None), a config filename, - a hook description, and whether this is a dry run, run the given commands. Or, don't run them - if this is a dry run. + Given a sequence of command hook dicts from configuration and one or more filters (before name, + after name, calling hook name, or a sequence of action names), filter down the command hooks to + just the ones that match the given filters. + ''' + return tuple( + hook_config + for hook_config in command_hooks or () + for config_action_names in (hook_config.get('when'),) + if before is None or hook_config.get('before') == before + if after is None or hook_config.get('after') == after + if action_names is None + or config_action_names is None + or set(config_action_names or ()).intersection(set(action_names)) + ) + + +def execute_hooks(command_hooks, umask, working_directory, dry_run, **context): + ''' + Given a sequence of command hook dicts from configuration, a umask to execute with (or None), a + working directory to execute with, and whether this is a dry run, run the commands for each + hook. Or don't run them if this is a dry run. The context contains optional values interpolated by name into the hook commands. - Raise ValueError if the umask cannot be parsed. + Raise ValueError if the umask cannot be parsed or a hook is invalid. Raise subprocesses.CalledProcessError if an error occurs in a hook. ''' - if not commands: - logger.debug(f'No commands to run for {description} hook') - return + borgmatic.logger.add_custom_log_levels() dry_run_label = ' (dry run; not actually running hooks)' if dry_run else '' - context['configuration_filename'] = config_filename - commands = [interpolate_context(description, command, context) for command in commands] + for hook_config in command_hooks: + commands = hook_config.get('run') - if len(commands) == 1: - logger.info(f'Running command for {description} hook{dry_run_label}') - else: - logger.info( - f'Running {len(commands)} commands for {description} hook{dry_run_label}', - ) + if 'before' in hook_config: + description = f'before {hook_config.get("before")}' + elif 'after' in hook_config: + description = f'after {hook_config.get("after")}' + else: + raise ValueError(f'Invalid hook configuration: {hook_config}') - if umask: - parsed_umask = int(str(umask), 8) - logger.debug(f'Set hook umask to {oct(parsed_umask)}') - original_umask = os.umask(parsed_umask) - else: - original_umask = None + if not commands: + logger.debug(f'No commands to run for {description} hook') + continue - try: - for command in commands: - if dry_run: - continue + commands = [interpolate_context(description, command, context) for command in commands] - borgmatic.execute.execute_command( - [command], - output_log_level=(logging.ERROR if description == 'on-error' else logging.WARNING), - shell=True, - extra_environment=make_environment(os.environ), + if len(commands) == 1: + logger.info(f'Running {description} command hook{dry_run_label}') + else: + logger.info( + f'Running {len(commands)} commands for {description} hook{dry_run_label}', ) - finally: - if original_umask: - os.umask(original_umask) + + if umask: + parsed_umask = int(str(umask), 8) + logger.debug(f'Setting hook umask to {oct(parsed_umask)}') + original_umask = os.umask(parsed_umask) + else: + original_umask = None + + try: + for command in commands: + if dry_run: + continue + + borgmatic.execute.execute_command( + [command], + output_log_level=( + logging.ERROR if hook_config.get('after') == 'error' else logging.ANSWER + ), + shell=True, + environment=make_environment(os.environ), + working_directory=working_directory, + ) + finally: + if original_umask: + os.umask(original_umask) + + +class Before_after_hooks: + ''' + A Python context manager for executing command hooks both before and after the wrapped code. + + Example use as a context manager: + + with borgmatic.hooks.command.Before_after_hooks( + command_hooks=config.get('commands'), + before_after='do_stuff', + umask=config.get('umask'), + dry_run=dry_run, + hook_name='myhook', + ): + do() + some() + stuff() + + With that context manager in place, "before" command hooks execute before the wrapped code runs, + and "after" command hooks execute after the wrapped code completes. + ''' + + def __init__( + self, + command_hooks, + before_after, + umask, + working_directory, + dry_run, + hook_name=None, + action_names=None, + **context, + ): + ''' + Given a sequence of command hook configuration dicts, the before/after name, a umask to run + commands with, a working directory to run commands with, a dry run flag, the name of the + calling hook, a sequence of action names, and any context for the executed commands, save + those data points for use below. + ''' + self.command_hooks = command_hooks + self.before_after = before_after + self.umask = umask + self.working_directory = working_directory + self.dry_run = dry_run + self.hook_name = hook_name + self.action_names = action_names + self.context = context + + def __enter__(self): + ''' + Run the configured "before" command hooks that match the initialized data points. + ''' + try: + execute_hooks( + borgmatic.hooks.command.filter_hooks( + self.command_hooks, + before=self.before_after, + hook_name=self.hook_name, + action_names=self.action_names, + ), + self.umask, + self.working_directory, + self.dry_run, + **self.context, + ) + except (OSError, subprocess.CalledProcessError) as error: + if considered_soft_failure(error): + return + + # Trigger the after hook manually, since raising here will prevent it from being run + # otherwise. + self.__exit__(None, None, None) + + raise ValueError(f'Error running before {self.before_after} hook: {error}') + + def __exit__(self, exception_type, exception, traceback): + ''' + Run the configured "after" command hooks that match the initialized data points. + ''' + try: + execute_hooks( + borgmatic.hooks.command.filter_hooks( + self.command_hooks, + after=self.before_after, + hook_name=self.hook_name, + action_names=self.action_names, + ), + self.umask, + self.working_directory, + self.dry_run, + **self.context, + ) + except (OSError, subprocess.CalledProcessError) as error: + if considered_soft_failure(error): + return + + raise ValueError(f'Error running after {self.before_after} hook: {error}') def considered_soft_failure(error): diff --git a/borgmatic/hooks/credential/container.py b/borgmatic/hooks/credential/container.py new file mode 100644 index 00000000..a38e6d35 --- /dev/null +++ b/borgmatic/hooks/credential/container.py @@ -0,0 +1,43 @@ +import logging +import os +import re + +logger = logging.getLogger(__name__) + + +SECRET_NAME_PATTERN = re.compile(r'^\w+$') +DEFAULT_SECRETS_DIRECTORY = '/run/secrets' + + +def load_credential(hook_config, config, credential_parameters): + ''' + Given the hook configuration dict, the configuration dict, and a credential parameters tuple + containing a secret name to load, read the secret from the corresponding container secrets file + and return it. + + Raise ValueError if the credential parameters is not one element, the secret name is invalid, or + the secret file cannot be read. + ''' + try: + (secret_name,) = credential_parameters + except ValueError: + name = ' '.join(credential_parameters) + + raise ValueError(f'Cannot load invalid secret name: "{name}"') + + if not SECRET_NAME_PATTERN.match(secret_name): + raise ValueError(f'Cannot load invalid secret name: "{secret_name}"') + + try: + with open( + os.path.join( + config.get('working_directory', ''), + (hook_config or {}).get('secrets_directory', DEFAULT_SECRETS_DIRECTORY), + secret_name, + ) + ) as secret_file: + return secret_file.read().rstrip(os.linesep) + except (FileNotFoundError, OSError) as error: + logger.warning(error) + + raise ValueError(f'Cannot load secret "{secret_name}" from file: {error.filename}') diff --git a/borgmatic/hooks/credential/file.py b/borgmatic/hooks/credential/file.py new file mode 100644 index 00000000..366a7d4a --- /dev/null +++ b/borgmatic/hooks/credential/file.py @@ -0,0 +1,32 @@ +import logging +import os + +logger = logging.getLogger(__name__) + + +def load_credential(hook_config, config, credential_parameters): + ''' + Given the hook configuration dict, the configuration dict, and a credential parameters tuple + containing a credential path to load, load the credential from file and return it. + + Raise ValueError if the credential parameters is not one element or the secret file cannot be + read. + ''' + try: + (credential_path,) = credential_parameters + except ValueError: + name = ' '.join(credential_parameters) + + raise ValueError(f'Cannot load invalid credential: "{name}"') + + expanded_credential_path = os.path.expanduser(credential_path) + + try: + with open( + os.path.join(config.get('working_directory', ''), expanded_credential_path) + ) as credential_file: + return credential_file.read().rstrip(os.linesep) + except (FileNotFoundError, OSError) as error: + logger.warning(error) + + raise ValueError(f'Cannot load credential file: {error.filename}') diff --git a/borgmatic/hooks/credential/keepassxc.py b/borgmatic/hooks/credential/keepassxc.py new file mode 100644 index 00000000..c3605fcf --- /dev/null +++ b/borgmatic/hooks/credential/keepassxc.py @@ -0,0 +1,45 @@ +import logging +import os +import shlex + +import borgmatic.execute + +logger = logging.getLogger(__name__) + + +def load_credential(hook_config, config, credential_parameters): + ''' + Given the hook configuration dict, the configuration dict, and a credential parameters tuple + containing a KeePassXC database path and an attribute name to load, run keepassxc-cli to fetch + the corresponding KeePassXC credential and return it. + + Raise ValueError if keepassxc-cli can't retrieve the credential. + ''' + try: + (database_path, attribute_name) = credential_parameters + except ValueError: + raise ValueError(f'Invalid KeePassXC credential parameters: {credential_parameters}') + + expanded_database_path = os.path.expanduser(database_path) + + if not os.path.exists(expanded_database_path): + raise ValueError(f'KeePassXC database path does not exist: {database_path}') + + # Build the keepassxc-cli command. + command = ( + tuple(shlex.split((hook_config or {}).get('keepassxc_cli_command', 'keepassxc-cli'))) + + ('show', '--show-protected', '--attributes', 'Password') + + ( + ('--key-file', hook_config['key_file']) + if hook_config and hook_config.get('key_file') + else () + ) + + ( + ('--yubikey', hook_config['yubikey']) + if hook_config and hook_config.get('yubikey') + else () + ) + + (expanded_database_path, attribute_name) # Ensure database and entry are last. + ) + + return borgmatic.execute.execute_command_and_capture_output(command).rstrip(os.linesep) diff --git a/borgmatic/hooks/credential/parse.py b/borgmatic/hooks/credential/parse.py new file mode 100644 index 00000000..2b7b2543 --- /dev/null +++ b/borgmatic/hooks/credential/parse.py @@ -0,0 +1,124 @@ +import functools +import re +import shlex + +import borgmatic.hooks.dispatch + +IS_A_HOOK = False + + +class Hash_adapter: + ''' + A Hash_adapter instance wraps an unhashable object and pretends it's hashable. This is intended + for passing to a @functools.cache-decorated function to prevent it from complaining that an + argument is unhashable. It should only be used for arguments that you don't want to actually + impact the cache hashing, because Hash_adapter doesn't actually hash the object's contents. + + Example usage: + + @functools.cache + def func(a, b): + print(a, b.actual_value) + return a + + func(5, Hash_adapter({1: 2, 3: 4})) # Calls func(), prints, and returns. + func(5, Hash_adapter({1: 2, 3: 4})) # Hits the cache and just returns the value. + func(5, Hash_adapter({5: 6, 7: 8})) # Also uses cache, since the Hash_adapter is ignored. + + In the above function, the "b" value is one that has been wrapped with Hash_adappter, and + therefore "b.actual_value" is necessary to access the original value. + ''' + + def __init__(self, actual_value): + self.actual_value = actual_value + + def __eq__(self, other): + return True + + def __hash__(self): + return 0 + + +UNHASHABLE_TYPES = (dict, list, set) + + +def cache_ignoring_unhashable_arguments(function): + ''' + A function decorator that caches calls to the decorated function but ignores any unhashable + arguments when performing cache lookups. This is intended to be a drop-in replacement for + functools.cache. + + Example usage: + + @cache_ignoring_unhashable_arguments + def func(a, b): + print(a, b) + return a + + func(5, {1: 2, 3: 4}) # Calls func(), prints, and returns. + func(5, {1: 2, 3: 4}) # Hits the cache and just returns the value. + func(5, {5: 6, 7: 8}) # Also uses cache, since the unhashable value (the dict) is ignored. + ''' + + @functools.cache + def cached_function(*args, **kwargs): + return function( + *(arg.actual_value if isinstance(arg, Hash_adapter) else arg for arg in args), + **{ + key: value.actual_value if isinstance(value, Hash_adapter) else value + for (key, value) in kwargs.items() + }, + ) + + @functools.wraps(function) + def wrapper_function(*args, **kwargs): + return cached_function( + *(Hash_adapter(arg) if isinstance(arg, UNHASHABLE_TYPES) else arg for arg in args), + **{ + key: Hash_adapter(value) if isinstance(value, UNHASHABLE_TYPES) else value + for (key, value) in kwargs.items() + }, + ) + + wrapper_function.cache_clear = cached_function.cache_clear + + return wrapper_function + + +CREDENTIAL_PATTERN = re.compile(r'\{credential( +(?P.*))?\}') + + +@cache_ignoring_unhashable_arguments +def resolve_credential(value, config): + ''' + Given a configuration value containing a string like "{credential hookname credentialname}" and + a configuration dict, resolve the credential by calling the relevant hook to get the actual + credential value. If the given value does not actually contain a credential tag, then return it + unchanged. + + Cache the value (ignoring the config for purposes of caching), so repeated calls to this + function don't need to load the credential repeatedly. + + Raise ValueError if the config could not be parsed or the credential could not be loaded. + ''' + if value is None: + return value + + matcher = CREDENTIAL_PATTERN.match(value) + + if not matcher: + return value + + hook_and_parameters = matcher.group('hook_and_parameters') + + if not hook_and_parameters: + raise ValueError(f'Cannot load credential with invalid syntax "{value}"') + + (hook_name, *credential_parameters) = shlex.split(hook_and_parameters) + + if not credential_parameters: + raise ValueError(f'Cannot load credential with invalid syntax "{value}"') + + return borgmatic.hooks.dispatch.call_hook( + 'load_credential', config, hook_name, tuple(credential_parameters) + ) diff --git a/borgmatic/hooks/credential/systemd.py b/borgmatic/hooks/credential/systemd.py new file mode 100644 index 00000000..9cce0d7b --- /dev/null +++ b/borgmatic/hooks/credential/systemd.py @@ -0,0 +1,43 @@ +import logging +import os +import re + +logger = logging.getLogger(__name__) + + +CREDENTIAL_NAME_PATTERN = re.compile(r'^[\w.-]+$') + + +def load_credential(hook_config, config, credential_parameters): + ''' + Given the hook configuration dict, the configuration dict, and a credential parameters tuple + containing a credential name to load, read the credential from the corresponding systemd + credential file and return it. + + Raise ValueError if the systemd CREDENTIALS_DIRECTORY environment variable is not set, the + credential name is invalid, or the credential file cannot be read. + ''' + try: + (credential_name,) = credential_parameters + except ValueError: + name = ' '.join(credential_parameters) + + raise ValueError(f'Cannot load invalid credential name: "{name}"') + + credentials_directory = os.environ.get('CREDENTIALS_DIRECTORY') + + if not credentials_directory: + raise ValueError( + f'Cannot load credential "{credential_name}" because the systemd CREDENTIALS_DIRECTORY environment variable is not set' + ) + + if not CREDENTIAL_NAME_PATTERN.match(credential_name): + raise ValueError(f'Cannot load invalid credential name "{credential_name}"') + + try: + with open(os.path.join(credentials_directory, credential_name)) as credential_file: + return credential_file.read().rstrip(os.linesep) + except (FileNotFoundError, OSError) as error: + logger.warning(error) + + raise ValueError(f'Cannot load credential "{credential_name}" from file: {error.filename}') diff --git a/borgmatic/hooks/data_source/bootstrap.py b/borgmatic/hooks/data_source/bootstrap.py index 67d75421..fe779f0e 100644 --- a/borgmatic/hooks/data_source/bootstrap.py +++ b/borgmatic/hooks/data_source/bootstrap.py @@ -55,9 +55,17 @@ def dump_data_sources( manifest_file, ) - patterns.extend(borgmatic.borg.pattern.Pattern(config_path) for config_path in config_paths) + patterns.extend( + borgmatic.borg.pattern.Pattern( + config_path, source=borgmatic.borg.pattern.Pattern_source.HOOK + ) + for config_path in config_paths + ) patterns.append( - borgmatic.borg.pattern.Pattern(os.path.join(borgmatic_runtime_directory, 'bootstrap')) + borgmatic.borg.pattern.Pattern( + os.path.join(borgmatic_runtime_directory, 'bootstrap'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, + ) ) return [] diff --git a/borgmatic/hooks/data_source/btrfs.py b/borgmatic/hooks/data_source/btrfs.py index a7d8f2f5..f46cbc2e 100644 --- a/borgmatic/hooks/data_source/btrfs.py +++ b/borgmatic/hooks/data_source/btrfs.py @@ -48,13 +48,56 @@ def get_subvolume_mount_points(findmnt_command): Subvolume = collections.namedtuple('Subvolume', ('path', 'contained_patterns'), defaults=((),)) +def get_subvolume_property(btrfs_command, subvolume_path, property_name): + output = borgmatic.execute.execute_command_and_capture_output( + tuple(btrfs_command.split(' ')) + + ( + 'property', + 'get', + '-t', # Type. + 'subvol', + subvolume_path, + property_name, + ), + ) + + try: + value = output.strip().split('=')[1] + except IndexError: + raise ValueError(f'Invalid {btrfs_command} property output') + + return { + 'true': True, + 'false': False, + }.get(value, value) + + +def omit_read_only_subvolume_mount_points(btrfs_command, subvolume_paths): + ''' + Given a Btrfs command to run and a sequence of Btrfs subvolume mount points, filter them down to + just those that are read-write. The idea is that Btrfs can't actually snapshot a read-only + subvolume, so we should just ignore them. + ''' + retained_subvolume_paths = [] + + for subvolume_path in subvolume_paths: + if get_subvolume_property(btrfs_command, subvolume_path, 'ro'): + logger.debug(f'Ignoring Btrfs subvolume {subvolume_path} because it is read-only') + else: + retained_subvolume_paths.append(subvolume_path) + + return tuple(retained_subvolume_paths) + + def get_subvolumes(btrfs_command, findmnt_command, patterns=None): ''' Given a Btrfs command to run and a sequence of configured patterns, find the intersection between the current Btrfs filesystem and subvolume mount points and the paths of any patterns. The idea is that these pattern paths represent the requested subvolumes to snapshot. - If patterns is None, then return all subvolumes, sorted by path. + Only include subvolumes that contain at least one root pattern sourced from borgmatic + configuration (as opposed to generated elsewhere in borgmatic). But if patterns is None, then + return all subvolumes instead, sorted by path. Return the result as a sequence of matching subvolume mount points. ''' @@ -65,7 +108,11 @@ def get_subvolumes(btrfs_command, findmnt_command, patterns=None): # backup. Sort the subvolumes from longest to shortest mount points, so longer mount points get # a whack at the candidate pattern piñata before their parents do. (Patterns are consumed during # this process, so no two subvolumes end up with the same contained patterns.) - for mount_point in reversed(get_subvolume_mount_points(findmnt_command)): + for mount_point in reversed( + omit_read_only_subvolume_mount_points( + btrfs_command, get_subvolume_mount_points(findmnt_command) + ) + ): subvolumes.extend( Subvolume(mount_point, contained_patterns) for contained_patterns in ( @@ -73,7 +120,12 @@ def get_subvolumes(btrfs_command, findmnt_command, patterns=None): mount_point, candidate_patterns ), ) - if patterns is None or contained_patterns + if patterns is None + or any( + pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT + and pattern.source == borgmatic.borg.pattern.Pattern_source.CONFIG + for pattern in contained_patterns + ) ) return tuple(sorted(subvolumes, key=lambda subvolume: subvolume.path)) @@ -121,6 +173,7 @@ def make_snapshot_exclude_pattern(subvolume_path): # pragma: no cover ), borgmatic.borg.pattern.Pattern_type.NO_RECURSE, borgmatic.borg.pattern.Pattern_style.FNMATCH, + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) @@ -153,6 +206,7 @@ def make_borg_snapshot_pattern(subvolume_path, pattern): pattern.type, pattern.style, pattern.device, + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) @@ -198,7 +252,8 @@ def dump_data_sources( dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' logger.info(f'Snapshotting Btrfs subvolumes{dry_run_label}') - # Based on the configured patterns, determine Btrfs subvolumes to backup. + # Based on the configured patterns, determine Btrfs subvolumes to backup. Only consider those + # patterns that came from actual user configuration (as opposed to, say, other hooks). btrfs_command = hook_config.get('btrfs_command', 'btrfs') findmnt_command = hook_config.get('findmnt_command', 'findmnt') subvolumes = get_subvolumes(btrfs_command, findmnt_command, patterns) @@ -299,9 +354,12 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d logger.debug(error) return - # Strip off the subvolume path from the end of the snapshot path and then delete the - # resulting directory. - shutil.rmtree(snapshot_path.rsplit(subvolume.path, 1)[0]) + # Remove the snapshot parent directory if it still exists. (It might not exist if the + # snapshot was for "/".) + snapshot_parent_dir = snapshot_path.rsplit(subvolume.path, 1)[0] + + if os.path.isdir(snapshot_parent_dir): + shutil.rmtree(snapshot_parent_dir) def make_data_source_dump_patterns( diff --git a/borgmatic/hooks/data_source/lvm.py b/borgmatic/hooks/data_source/lvm.py index 8436d0f6..11cd1c09 100644 --- a/borgmatic/hooks/data_source/lvm.py +++ b/borgmatic/hooks/data_source/lvm.py @@ -1,5 +1,6 @@ import collections import glob +import hashlib import json import logging import os @@ -33,7 +34,9 @@ def get_logical_volumes(lsblk_command, patterns=None): between the current LVM logical volume mount points and the paths of any patterns. The idea is that these pattern paths represent the requested logical volumes to snapshot. - If patterns is None, include all logical volume mounts points, not just those in patterns. + Only include logical volumes that contain at least one root pattern sourced from borgmatic + configuration (as opposed to generated elsewhere in borgmatic). But if patterns is None, include + all logical volume mounts points instead, not just those in patterns. Return the result as a sequence of Logical_volume instances. ''' @@ -72,7 +75,12 @@ def get_logical_volumes(lsblk_command, patterns=None): device['mountpoint'], candidate_patterns ), ) - if not patterns or contained_patterns + if not patterns + or any( + pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT + and pattern.source == borgmatic.borg.pattern.Pattern_source.CONFIG + for pattern in contained_patterns + ) ) except KeyError as error: raise ValueError(f'Invalid {lsblk_command} output: Missing key "{error}"') @@ -124,10 +132,14 @@ def mount_snapshot(mount_command, snapshot_device, snapshot_mount_path): # prag ) -def make_borg_snapshot_pattern(pattern, normalized_runtime_directory): +MOUNT_POINT_HASH_LENGTH = 10 + + +def make_borg_snapshot_pattern(pattern, logical_volume, normalized_runtime_directory): ''' - Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance, return a new Pattern with its - path rewritten to be in a snapshot directory based on the given runtime directory. + Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance and a Logical_volume + containing it, return a new Pattern with its path rewritten to be in a snapshot directory based + on both the given runtime directory and the given Logical_volume's mount point. Move any initial caret in a regular expression pattern path to the beginning, so as not to break the regular expression. @@ -142,6 +154,13 @@ def make_borg_snapshot_pattern(pattern, normalized_runtime_directory): rewritten_path = initial_caret + os.path.join( normalized_runtime_directory, 'lvm_snapshots', + # Including this hash prevents conflicts between snapshot patterns for different logical + # volumes. For instance, without this, snapshotting a logical volume at /var and another at + # /var/spool would result in overlapping snapshot patterns and therefore colliding mount + # attempts. + hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest( + MOUNT_POINT_HASH_LENGTH + ), '.', # Borg 1.4+ "slashdot" hack. # Included so that the source directory ends up in the Borg archive at its "original" path. pattern.path.lstrip('^').lstrip(os.path.sep), @@ -152,6 +171,7 @@ def make_borg_snapshot_pattern(pattern, normalized_runtime_directory): pattern.type, pattern.style, pattern.device, + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) @@ -180,7 +200,8 @@ def dump_data_sources( dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' logger.info(f'Snapshotting LVM logical volumes{dry_run_label}') - # List logical volumes to get their mount points. + # List logical volumes to get their mount points, but only consider those patterns that came + # from actual user configuration (as opposed to, say, other hooks). lsblk_command = hook_config.get('lsblk_command', 'lsblk') requested_logical_volumes = get_logical_volumes(lsblk_command, patterns) @@ -218,6 +239,9 @@ def dump_data_sources( snapshot_mount_path = os.path.join( normalized_runtime_directory, 'lvm_snapshots', + hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest( + MOUNT_POINT_HASH_LENGTH + ), logical_volume.mount_point.lstrip(os.path.sep), ) @@ -233,7 +257,9 @@ def dump_data_sources( ) for pattern in logical_volume.contained_patterns: - snapshot_pattern = make_borg_snapshot_pattern(pattern, normalized_runtime_directory) + snapshot_pattern = make_borg_snapshot_pattern( + pattern, logical_volume, normalized_runtime_directory + ) # Attempt to update the pattern in place, since pattern order matters to Borg. try: @@ -337,6 +363,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d os.path.normpath(borgmatic_runtime_directory), ), 'lvm_snapshots', + '*', ) logger.debug(f'Looking for snapshots to remove in {snapshots_glob}{dry_run_label}') umount_command = hook_config.get('umount_command', 'umount') @@ -349,7 +376,10 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d snapshot_mount_path = os.path.join( snapshots_directory, logical_volume.mount_point.lstrip(os.path.sep) ) - if not os.path.isdir(snapshot_mount_path): + + # If the snapshot mount path is empty, this is probably just a "shadow" of a nested + # logical volume and therefore there's nothing to unmount. + if not os.path.isdir(snapshot_mount_path) or not os.listdir(snapshot_mount_path): continue # This might fail if the directory is already mounted, but we swallow errors here since @@ -374,7 +404,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d return except subprocess.CalledProcessError as error: logger.debug(error) - return + continue if not dry_run: shutil.rmtree(snapshots_directory) diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index d309d9e5..1f140793 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -1,10 +1,12 @@ import copy import logging import os +import re import shlex import borgmatic.borg.pattern import borgmatic.config.paths +import borgmatic.hooks.credential.parse from borgmatic.execute import ( execute_command, execute_command_and_capture_output, @@ -22,14 +24,92 @@ def make_dump_path(base_directory): # pragma: no cover return dump.make_data_source_dump_path(base_directory, 'mariadb_databases') -SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys') +DEFAULTS_EXTRA_FILE_FLAG_PATTERN = re.compile('^--defaults-extra-file=(?P.*)$') -def database_names_to_dump(database, extra_environment, dry_run): +def parse_extra_options(extra_options): ''' - Given a requested database config, return the corresponding sequence of database names to dump. - In the case of "all", query for the names of databases on the configured host and return them, - excluding any system databases that will cause problems during restore. + Given an extra options string, split the options into a tuple and return it. Additionally, if + the first option is "--defaults-extra-file=...", then remove it from the options and return the + filename. + + So the return value is a tuple of: (parsed options, defaults extra filename). + + The intent is to support downstream merging of multiple "--defaults-extra-file"s, as + MariaDB/MySQL only allows one at a time. + ''' + split_extra_options = tuple(shlex.split(extra_options)) if extra_options else () + + if not split_extra_options: + return ((), None) + + match = DEFAULTS_EXTRA_FILE_FLAG_PATTERN.match(split_extra_options[0]) + + if not match: + return (split_extra_options, None) + + return (split_extra_options[1:], match.group('filename')) + + +def make_defaults_file_options(username=None, password=None, defaults_extra_filename=None): + ''' + Given a database username and/or password, write it to an anonymous pipe and return the flags + for passing that file descriptor to an executed command. The idea is that this is a more secure + way to transmit credentials to a database client than using an environment variable. + + If no username or password are given, then return the options for the given defaults extra + filename (if any). But if there is a username and/or password and a defaults extra filename is + given, then "!include" it from the generated file, effectively allowing multiple defaults extra + files. + + Do not use the returned value for multiple different command invocations. That will not work + because each pipe is "used up" once read. + ''' + escaped_password = None if password is None else password.replace('\\', '\\\\') + + values = '\n'.join( + ( + (f'user={username}' if username is not None else ''), + (f'password="{escaped_password}"' if escaped_password is not None else ''), + ) + ).strip() + + if not values: + if defaults_extra_filename: + return (f'--defaults-extra-file={defaults_extra_filename}',) + + return () + + fields_message = ' and '.join( + field_name + for field_name in ( + (f'username ({username})' if username is not None else None), + ('password' if password is not None else None), + ) + if field_name is not None + ) + include_message = f' (including {defaults_extra_filename})' if defaults_extra_filename else '' + logger.debug(f'Writing database {fields_message} to defaults extra file pipe{include_message}') + + include = f'!include {defaults_extra_filename}\n' if defaults_extra_filename else '' + + read_file_descriptor, write_file_descriptor = os.pipe() + os.write(write_file_descriptor, f'{include}[client]\n{values}'.encode('utf-8')) + os.close(write_file_descriptor) + + # This plus subprocess.Popen(..., close_fds=False) in execute.py is necessary for the database + # client child process to inherit the file descriptor. + os.set_inheritable(read_file_descriptor, True) + + return (f'--defaults-extra-file=/dev/fd/{read_file_descriptor}',) + + +def database_names_to_dump(database, config, username, password, environment, dry_run): + ''' + Given a requested database config, a configuration dict, a database username and password, an + environment dict, and whether this is a dry run, return the corresponding sequence of database + names to dump. In the case of "all", query for the names of databases on the configured host and + return them, excluding any system databases that will cause problems during restore. ''' if database['name'] != 'all': return (database['name'],) @@ -39,20 +119,23 @@ def database_names_to_dump(database, extra_environment, dry_run): mariadb_show_command = tuple( shlex.quote(part) for part in shlex.split(database.get('mariadb_command') or 'mariadb') ) + extra_options, defaults_extra_filename = parse_extra_options(database.get('list_options')) show_command = ( mariadb_show_command - + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ()) + + make_defaults_file_options(username, password, defaults_extra_filename) + + extra_options + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) - + (('--user', database['username']) if 'username' in database else ()) + + (('--ssl',) if database.get('tls') is True else ()) + + (('--skip-ssl',) if database.get('tls') is False else ()) + ('--skip-column-names', '--batch') + ('--execute', 'show schemas') ) + logger.debug('Querying for "all" MariaDB databases to dump') - show_output = execute_command_and_capture_output( - show_command, extra_environment=extra_environment - ) + + show_output = execute_command_and_capture_output(show_command, environment=environment) return tuple( show_name @@ -61,8 +144,19 @@ def database_names_to_dump(database, extra_environment, dry_run): ) +SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys') + + def execute_dump_command( - database, dump_path, database_names, extra_environment, dry_run, dry_run_label + database, + config, + username, + password, + dump_path, + database_names, + environment, + dry_run, + dry_run_label, ): ''' Kick off a dump for the given MariaDB database (provided as a configuration dict) to a named @@ -89,14 +183,17 @@ def execute_dump_command( shlex.quote(part) for part in shlex.split(database.get('mariadb_dump_command') or 'mariadb-dump') ) + extra_options, defaults_extra_filename = parse_extra_options(database.get('options')) dump_command = ( mariadb_dump_command - + (tuple(database['options'].split(' ')) if 'options' in database else ()) + + make_defaults_file_options(username, password, defaults_extra_filename) + + extra_options + (('--add-drop-database',) if database.get('add_drop_database', True) else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) - + (('--user', database['username']) if 'username' in database else ()) + + (('--ssl',) if database.get('tls') is True else ()) + + (('--skip-ssl',) if database.get('tls') is False else ()) + ('--databases',) + database_names + ('--result-file', dump_filename) @@ -110,7 +207,7 @@ def execute_dump_command( return execute_command( dump_command, - extra_environment=extra_environment, + environment=environment, run_to_completion=False, ) @@ -152,8 +249,16 @@ def dump_data_sources( for database in databases: dump_path = make_dump_path(borgmatic_runtime_directory) - extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None - dump_database_names = database_names_to_dump(database, extra_environment, dry_run) + username = borgmatic.hooks.credential.parse.resolve_credential( + database.get('username'), config + ) + password = borgmatic.hooks.credential.parse.resolve_credential( + database.get('password'), config + ) + environment = dict(os.environ) + dump_database_names = database_names_to_dump( + database, config, username, password, environment, dry_run + ) if not dump_database_names: if dry_run: @@ -168,9 +273,12 @@ def dump_data_sources( processes.append( execute_dump_command( renamed_database, + config, + username, + password, dump_path, (dump_name,), - extra_environment, + environment, dry_run, dry_run_label, ) @@ -179,9 +287,12 @@ def dump_data_sources( processes.append( execute_dump_command( database, + config, + username, + password, dump_path, dump_database_names, - extra_environment, + environment, dry_run, dry_run_label, ) @@ -190,7 +301,8 @@ def dump_data_sources( if not dry_run: patterns.append( borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'mariadb_databases') + os.path.join(borgmatic_runtime_directory, 'mariadb_databases'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) ) @@ -251,30 +363,38 @@ def restore_data_source_dump( port = str( connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')) ) - username = connection_params['username'] or data_source.get( - 'restore_username', data_source.get('username') + tls = data_source.get('restore_tls', data_source.get('tls')) + username = borgmatic.hooks.credential.parse.resolve_credential( + ( + connection_params['username'] + or data_source.get('restore_username', data_source.get('username')) + ), + config, ) - password = connection_params['password'] or data_source.get( - 'restore_password', data_source.get('password') + password = borgmatic.hooks.credential.parse.resolve_credential( + ( + connection_params['password'] + or data_source.get('restore_password', data_source.get('password')) + ), + config, ) mariadb_restore_command = tuple( shlex.quote(part) for part in shlex.split(data_source.get('mariadb_command') or 'mariadb') ) + extra_options, defaults_extra_filename = parse_extra_options(data_source.get('restore_options')) restore_command = ( mariadb_restore_command + + make_defaults_file_options(username, password, defaults_extra_filename) + + extra_options + ('--batch',) - + ( - tuple(data_source['restore_options'].split(' ')) - if 'restore_options' in data_source - else () - ) + (('--host', hostname) if hostname else ()) + (('--port', str(port)) if port else ()) + (('--protocol', 'tcp') if hostname or port else ()) - + (('--user', username) if username else ()) + + (('--ssl',) if tls is True else ()) + + (('--skip-ssl',) if tls is False else ()) ) - extra_environment = {'MYSQL_PWD': password} if password else None + environment = dict(os.environ) logger.debug(f"Restoring MariaDB database {data_source['name']}{dry_run_label}") if dry_run: @@ -287,5 +407,5 @@ def restore_data_source_dump( [extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment=extra_environment, + environment=environment, ) diff --git a/borgmatic/hooks/data_source/mongodb.py b/borgmatic/hooks/data_source/mongodb.py index a1a006a9..74ae15f3 100644 --- a/borgmatic/hooks/data_source/mongodb.py +++ b/borgmatic/hooks/data_source/mongodb.py @@ -4,6 +4,7 @@ import shlex import borgmatic.borg.pattern import borgmatic.config.paths +import borgmatic.hooks.credential.parse from borgmatic.execute import execute_command, execute_command_with_processes from borgmatic.hooks.data_source import dump @@ -52,6 +53,7 @@ def dump_data_sources( logger.info(f'Dumping MongoDB databases{dry_run_label}') processes = [] + for database in databases: name = database['name'] dump_filename = dump.make_data_source_dump_filename( @@ -68,7 +70,7 @@ def dump_data_sources( if dry_run: continue - command = build_dump_command(database, dump_filename, dump_format) + command = build_dump_command(database, config, dump_filename, dump_format) if dump_format == 'directory': dump.create_parent_directory_for_dump(dump_filename) @@ -80,26 +82,65 @@ def dump_data_sources( if not dry_run: patterns.append( borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'mongodb_databases') + os.path.join(borgmatic_runtime_directory, 'mongodb_databases'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) ) return processes -def build_dump_command(database, dump_filename, dump_format): +def make_password_config_file(password): ''' - Return the mongodump command from a single database configuration. + Given a database password, write it as a MongoDB configuration file to an anonymous pipe and + return its filename. The idea is that this is a more secure way to transmit a password to + MongoDB than providing it directly on the command-line. + + Do not use the returned value for multiple different command invocations. That will not work + because each pipe is "used up" once read. + ''' + logger.debug('Writing MongoDB password to configuration file pipe') + + read_file_descriptor, write_file_descriptor = os.pipe() + os.write(write_file_descriptor, f'password: {password}'.encode('utf-8')) + os.close(write_file_descriptor) + + # This plus subprocess.Popen(..., close_fds=False) in execute.py is necessary for the database + # client child process to inherit the file descriptor. + os.set_inheritable(read_file_descriptor, True) + + return f'/dev/fd/{read_file_descriptor}' + + +def build_dump_command(database, config, dump_filename, dump_format): + ''' + Return the custom mongodump_command from a single database configuration. ''' all_databases = database['name'] == 'all' + password = borgmatic.hooks.credential.parse.resolve_credential(database.get('password'), config) + + dump_command = tuple( + shlex.quote(part) for part in shlex.split(database.get('mongodump_command') or 'mongodump') + ) return ( - ('mongodump',) + dump_command + (('--out', shlex.quote(dump_filename)) if dump_format == 'directory' else ()) + (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ()) + (('--port', shlex.quote(str(database['port']))) if 'port' in database else ()) - + (('--username', shlex.quote(database['username'])) if 'username' in database else ()) - + (('--password', shlex.quote(database['password'])) if 'password' in database else ()) + + ( + ( + '--username', + shlex.quote( + borgmatic.hooks.credential.parse.resolve_credential( + database['username'], config + ) + ), + ) + if 'username' in database + else () + ) + + (('--config', make_password_config_file(password)) if password else ()) + ( ('--authenticationDatabase', shlex.quote(database['authentication_database'])) if 'authentication_database' in database @@ -173,7 +214,7 @@ def restore_data_source_dump( data_source.get('hostname'), ) restore_command = build_restore_command( - extract_process, data_source, dump_filename, connection_params + extract_process, data_source, config, dump_filename, connection_params ) logger.debug(f"Restoring MongoDB database {data_source['name']}{dry_run_label}") @@ -190,22 +231,33 @@ def restore_data_source_dump( ) -def build_restore_command(extract_process, database, dump_filename, connection_params): +def build_restore_command(extract_process, database, config, dump_filename, connection_params): ''' - Return the mongorestore command from a single database configuration. + Return the custom mongorestore_command from a single database configuration. ''' hostname = connection_params['hostname'] or database.get( 'restore_hostname', database.get('hostname') ) port = str(connection_params['port'] or database.get('restore_port', database.get('port', ''))) - username = connection_params['username'] or database.get( - 'restore_username', database.get('username') + username = borgmatic.hooks.credential.parse.resolve_credential( + ( + connection_params['username'] + or database.get('restore_username', database.get('username')) + ), + config, ) - password = connection_params['password'] or database.get( - 'restore_password', database.get('password') + password = borgmatic.hooks.credential.parse.resolve_credential( + ( + connection_params['password'] + or database.get('restore_password', database.get('password')) + ), + config, ) - command = ['mongorestore'] + command = list( + shlex.quote(part) + for part in shlex.split(database.get('mongorestore_command') or 'mongorestore') + ) if extract_process: command.append('--archive') else: @@ -219,7 +271,7 @@ def build_restore_command(extract_process, database, dump_filename, connection_p if username: command.extend(('--username', username)) if password: - command.extend(('--password', password)) + command.extend(('--config', make_password_config_file(password))) if 'authentication_database' in database: command.extend(('--authenticationDatabase', database['authentication_database'])) if 'restore_options' in database: diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index a477d2ac..7c5f84f2 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -5,6 +5,8 @@ import shlex import borgmatic.borg.pattern import borgmatic.config.paths +import borgmatic.hooks.credential.parse +import borgmatic.hooks.data_source.mariadb from borgmatic.execute import ( execute_command, execute_command_and_capture_output, @@ -25,11 +27,12 @@ def make_dump_path(base_directory): # pragma: no cover SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys') -def database_names_to_dump(database, extra_environment, dry_run): +def database_names_to_dump(database, config, username, password, environment, dry_run): ''' - Given a requested database config, return the corresponding sequence of database names to dump. - In the case of "all", query for the names of databases on the configured host and return them, - excluding any system databases that will cause problems during restore. + Given a requested database config, a configuration dict, a database username and password, an + environment dict, and whether this is a dry run, return the corresponding sequence of database + names to dump. In the case of "all", query for the names of databases on the configured host and + return them, excluding any system databases that will cause problems during restore. ''' if database['name'] != 'all': return (database['name'],) @@ -39,20 +42,27 @@ def database_names_to_dump(database, extra_environment, dry_run): mysql_show_command = tuple( shlex.quote(part) for part in shlex.split(database.get('mysql_command') or 'mysql') ) + extra_options, defaults_extra_filename = ( + borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('list_options')) + ) show_command = ( mysql_show_command - + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ()) + + borgmatic.hooks.data_source.mariadb.make_defaults_file_options( + username, password, defaults_extra_filename + ) + + extra_options + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) - + (('--user', database['username']) if 'username' in database else ()) + + (('--ssl',) if database.get('tls') is True else ()) + + (('--skip-ssl',) if database.get('tls') is False else ()) + ('--skip-column-names', '--batch') + ('--execute', 'show schemas') ) + logger.debug('Querying for "all" MySQL databases to dump') - show_output = execute_command_and_capture_output( - show_command, extra_environment=extra_environment - ) + + show_output = execute_command_and_capture_output(show_command, environment=environment) return tuple( show_name @@ -62,7 +72,15 @@ def database_names_to_dump(database, extra_environment, dry_run): def execute_dump_command( - database, dump_path, database_names, extra_environment, dry_run, dry_run_label + database, + config, + username, + password, + dump_path, + database_names, + environment, + dry_run, + dry_run_label, ): ''' Kick off a dump for the given MySQL/MariaDB database (provided as a configuration dict) to a @@ -88,14 +106,21 @@ def execute_dump_command( mysql_dump_command = tuple( shlex.quote(part) for part in shlex.split(database.get('mysql_dump_command') or 'mysqldump') ) + extra_options, defaults_extra_filename = ( + borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('options')) + ) dump_command = ( mysql_dump_command - + (tuple(database['options'].split(' ')) if 'options' in database else ()) + + borgmatic.hooks.data_source.mariadb.make_defaults_file_options( + username, password, defaults_extra_filename + ) + + extra_options + (('--add-drop-database',) if database.get('add_drop_database', True) else ()) + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ()) - + (('--user', database['username']) if 'username' in database else ()) + + (('--ssl',) if database.get('tls') is True else ()) + + (('--skip-ssl',) if database.get('tls') is False else ()) + ('--databases',) + database_names + ('--result-file', dump_filename) @@ -109,7 +134,7 @@ def execute_dump_command( return execute_command( dump_command, - extra_environment=extra_environment, + environment=environment, run_to_completion=False, ) @@ -151,8 +176,16 @@ def dump_data_sources( for database in databases: dump_path = make_dump_path(borgmatic_runtime_directory) - extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None - dump_database_names = database_names_to_dump(database, extra_environment, dry_run) + username = borgmatic.hooks.credential.parse.resolve_credential( + database.get('username'), config + ) + password = borgmatic.hooks.credential.parse.resolve_credential( + database.get('password'), config + ) + environment = dict(os.environ) + dump_database_names = database_names_to_dump( + database, config, username, password, environment, dry_run + ) if not dump_database_names: if dry_run: @@ -167,9 +200,12 @@ def dump_data_sources( processes.append( execute_dump_command( renamed_database, + config, + username, + password, dump_path, (dump_name,), - extra_environment, + environment, dry_run, dry_run_label, ) @@ -178,9 +214,12 @@ def dump_data_sources( processes.append( execute_dump_command( database, + config, + username, + password, dump_path, dump_database_names, - extra_environment, + environment, dry_run, dry_run_label, ) @@ -189,7 +228,8 @@ def dump_data_sources( if not dry_run: patterns.append( borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'mysql_databases') + os.path.join(borgmatic_runtime_directory, 'mysql_databases'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) ) @@ -250,30 +290,42 @@ def restore_data_source_dump( port = str( connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')) ) - username = connection_params['username'] or data_source.get( - 'restore_username', data_source.get('username') + tls = data_source.get('restore_tls', data_source.get('tls')) + username = borgmatic.hooks.credential.parse.resolve_credential( + ( + connection_params['username'] + or data_source.get('restore_username', data_source.get('username')) + ), + config, ) - password = connection_params['password'] or data_source.get( - 'restore_password', data_source.get('password') + password = borgmatic.hooks.credential.parse.resolve_credential( + ( + connection_params['password'] + or data_source.get('restore_password', data_source.get('password')) + ), + config, ) mysql_restore_command = tuple( shlex.quote(part) for part in shlex.split(data_source.get('mysql_command') or 'mysql') ) + extra_options, defaults_extra_filename = ( + borgmatic.hooks.data_source.mariadb.parse_extra_options(data_source.get('restore_options')) + ) restore_command = ( mysql_restore_command - + ('--batch',) - + ( - tuple(data_source['restore_options'].split(' ')) - if 'restore_options' in data_source - else () + + borgmatic.hooks.data_source.mariadb.make_defaults_file_options( + username, password, defaults_extra_filename ) + + extra_options + + ('--batch',) + (('--host', hostname) if hostname else ()) + (('--port', str(port)) if port else ()) + (('--protocol', 'tcp') if hostname or port else ()) - + (('--user', username) if username else ()) + + (('--ssl',) if tls is True else ()) + + (('--skip-ssl',) if tls is False else ()) ) - extra_environment = {'MYSQL_PWD': password} if password else None + environment = dict(os.environ) logger.debug(f"Restoring MySQL database {data_source['name']}{dry_run_label}") if dry_run: @@ -286,5 +338,5 @@ def restore_data_source_dump( [extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment=extra_environment, + environment=environment, ) diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index 40c3e57b..4f470c52 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -7,6 +7,7 @@ import shlex import borgmatic.borg.pattern import borgmatic.config.paths +import borgmatic.hooks.credential.parse from borgmatic.execute import ( execute_command, execute_command_and_capture_output, @@ -24,46 +25,52 @@ def make_dump_path(base_directory): # pragma: no cover return dump.make_data_source_dump_path(base_directory, 'postgresql_databases') -def make_extra_environment(database, restore_connection_params=None): +def make_environment(database, config, restore_connection_params=None): ''' - Make the extra_environment dict from the given database configuration. If restore connection - params are given, this is for a restore operation. + Make an environment dict from the current environment variables and the given database + configuration. If restore connection params are given, this is for a restore operation. ''' - extra = dict() + environment = dict(os.environ) try: if restore_connection_params: - extra['PGPASSWORD'] = restore_connection_params.get('password') or database.get( - 'restore_password', database['password'] + environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential( + ( + restore_connection_params.get('password') + or database.get('restore_password', database['password']) + ), + config, ) else: - extra['PGPASSWORD'] = database['password'] + environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential( + database['password'], config + ) except (AttributeError, KeyError): pass if 'ssl_mode' in database: - extra['PGSSLMODE'] = database['ssl_mode'] + environment['PGSSLMODE'] = database['ssl_mode'] if 'ssl_cert' in database: - extra['PGSSLCERT'] = database['ssl_cert'] + environment['PGSSLCERT'] = database['ssl_cert'] if 'ssl_key' in database: - extra['PGSSLKEY'] = database['ssl_key'] + environment['PGSSLKEY'] = database['ssl_key'] if 'ssl_root_cert' in database: - extra['PGSSLROOTCERT'] = database['ssl_root_cert'] + environment['PGSSLROOTCERT'] = database['ssl_root_cert'] if 'ssl_crl' in database: - extra['PGSSLCRL'] = database['ssl_crl'] + environment['PGSSLCRL'] = database['ssl_crl'] - return extra + return environment EXCLUDED_DATABASE_NAMES = ('template0', 'template1') -def database_names_to_dump(database, extra_environment, dry_run): +def database_names_to_dump(database, config, environment, dry_run): ''' - Given a requested database config, return the corresponding sequence of database names to dump. - In the case of "all" when a database format is given, query for the names of databases on the - configured host and return them. For "all" without a database format, just return a sequence - containing "all". + Given a requested database config and a configuration dict, return the corresponding sequence of + database names to dump. In the case of "all" when a database format is given, query for the + names of databases on the configured host and return them. For "all" without a database format, + just return a sequence containing "all". ''' requested_name = database['name'] @@ -82,13 +89,18 @@ def database_names_to_dump(database, extra_environment, dry_run): + ('--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only') + (('--host', database['hostname']) if 'hostname' in database else ()) + (('--port', str(database['port'])) if 'port' in database else ()) - + (('--username', database['username']) if 'username' in database else ()) + + ( + ( + '--username', + borgmatic.hooks.credential.parse.resolve_credential(database['username'], config), + ) + if 'username' in database + else () + ) + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ()) ) logger.debug('Querying for "all" PostgreSQL databases to dump') - list_output = execute_command_and_capture_output( - list_command, extra_environment=extra_environment - ) + list_output = execute_command_and_capture_output(list_command, environment=environment) return tuple( row[0] @@ -135,9 +147,9 @@ def dump_data_sources( logger.info(f'Dumping PostgreSQL databases{dry_run_label}') for database in databases: - extra_environment = make_extra_environment(database) + environment = make_environment(database, config) dump_path = make_dump_path(borgmatic_runtime_directory) - dump_database_names = database_names_to_dump(database, extra_environment, dry_run) + dump_database_names = database_names_to_dump(database, config, environment, dry_run) if not dump_database_names: if dry_run: @@ -147,6 +159,7 @@ def dump_data_sources( for database_name in dump_database_names: dump_format = database.get('format', None if database_name == 'all' else 'custom') + compression = database.get('compression') default_dump_command = 'pg_dumpall' if database_name == 'all' else 'pg_dump' dump_command = tuple( shlex.quote(part) @@ -174,12 +187,20 @@ def dump_data_sources( + (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ()) + (('--port', shlex.quote(str(database['port']))) if 'port' in database else ()) + ( - ('--username', shlex.quote(database['username'])) + ( + '--username', + shlex.quote( + borgmatic.hooks.credential.parse.resolve_credential( + database['username'], config + ) + ), + ) if 'username' in database else () ) + (('--no-owner',) if database.get('no_owner', False) else ()) + (('--format', shlex.quote(dump_format)) if dump_format else ()) + + (('--compress', shlex.quote(str(compression))) if compression is not None else ()) + (('--file', shlex.quote(dump_filename)) if dump_format == 'directory' else ()) + ( tuple(shlex.quote(option) for option in database['options'].split(' ')) @@ -204,7 +225,7 @@ def dump_data_sources( execute_command( command, shell=True, - extra_environment=extra_environment, + environment=environment, ) else: dump.create_named_pipe_for_dump(dump_filename) @@ -212,7 +233,7 @@ def dump_data_sources( execute_command( command, shell=True, - extra_environment=extra_environment, + environment=environment, run_to_completion=False, ) ) @@ -220,7 +241,8 @@ def dump_data_sources( if not dry_run: patterns.append( borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'postgresql_databases') + os.path.join(borgmatic_runtime_directory, 'postgresql_databases'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) ) @@ -290,8 +312,12 @@ def restore_data_source_dump( port = str( connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')) ) - username = connection_params['username'] or data_source.get( - 'restore_username', data_source.get('username') + username = borgmatic.hooks.credential.parse.resolve_credential( + ( + connection_params['username'] + or data_source.get('restore_username', data_source.get('username')) + ), + config, ) all_databases = bool(data_source['name'] == 'all') @@ -344,9 +370,7 @@ def restore_data_source_dump( ) ) - extra_environment = make_extra_environment( - data_source, restore_connection_params=connection_params - ) + environment = make_environment(data_source, config, restore_connection_params=connection_params) logger.debug(f"Restoring PostgreSQL database {data_source['name']}{dry_run_label}") if dry_run: @@ -359,6 +383,6 @@ def restore_data_source_dump( [extract_process] if extract_process else [], output_log_level=logging.DEBUG, input_file=extract_process.stdout if extract_process else None, - extra_environment=extra_environment, + environment=environment, ) - execute_command(analyze_command, extra_environment=extra_environment) + execute_command(analyze_command, environment=environment) diff --git a/borgmatic/hooks/data_source/snapshot.py b/borgmatic/hooks/data_source/snapshot.py index 2ee08a8f..a89b85df 100644 --- a/borgmatic/hooks/data_source/snapshot.py +++ b/borgmatic/hooks/data_source/snapshot.py @@ -1,3 +1,4 @@ +import os import pathlib IS_A_HOOK = False @@ -11,10 +12,14 @@ def get_contained_patterns(parent_directory, candidate_patterns): paths, but there's a parent directory (logical volume, dataset, subvolume, etc.) at /var, then /var is what we want to snapshot. - For this to work, a candidate pattern path can't have any globs or other non-literal characters - in the initial portion of the path that matches the parent directory. For instance, a parent - directory of /var would match a candidate pattern path of /var/log/*/data, but not a pattern - path like /v*/log/*/data. + If a parent directory and a candidate pattern are on different devices, skip the pattern. That's + because any snapshot of a parent directory won't actually include "contained" directories if + they reside on separate devices. + + For this function to work, a candidate pattern path can't have any globs or other non-literal + characters in the initial portion of the path that matches the parent directory. For instance, a + parent directory of /var would match a candidate pattern path of /var/log/*/data, but not a + pattern path like /v*/log/*/data. The one exception is that if a regular expression pattern path starts with "^", that will get stripped off for purposes of matching against a parent directory. @@ -27,12 +32,17 @@ def get_contained_patterns(parent_directory, candidate_patterns): if not candidate_patterns: return () + parent_device = os.stat(parent_directory).st_dev if os.path.exists(parent_directory) else None + contained_patterns = tuple( candidate for candidate in candidate_patterns for candidate_path in (pathlib.PurePath(candidate.path.lstrip('^')),) - if pathlib.PurePath(parent_directory) == candidate_path - or pathlib.PurePath(parent_directory) in candidate_path.parents + if ( + pathlib.PurePath(parent_directory) == candidate_path + or pathlib.PurePath(parent_directory) in candidate_path.parents + ) + if candidate.device == parent_device ) candidate_patterns -= set(contained_patterns) diff --git a/borgmatic/hooks/data_source/sqlite.py b/borgmatic/hooks/data_source/sqlite.py index 4d5ce9a0..347ca5dc 100644 --- a/borgmatic/hooks/data_source/sqlite.py +++ b/borgmatic/hooks/data_source/sqlite.py @@ -71,13 +71,16 @@ def dump_data_sources( ) continue - command = ( - 'sqlite3', + sqlite_command = tuple( + shlex.quote(part) for part in shlex.split(database.get('sqlite_command') or 'sqlite3') + ) + command = sqlite_command + ( shlex.quote(database_path), '.dump', '>', shlex.quote(dump_filename), ) + logger.debug( f'Dumping SQLite database at {database_path} to {dump_filename}{dry_run_label}' ) @@ -90,7 +93,8 @@ def dump_data_sources( if not dry_run: patterns.append( borgmatic.borg.pattern.Pattern( - os.path.join(borgmatic_runtime_directory, 'sqlite_databases') + os.path.join(borgmatic_runtime_directory, 'sqlite_databases'), + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) ) @@ -159,11 +163,11 @@ def restore_data_source_dump( except FileNotFoundError: # pragma: no cover pass - restore_command = ( - 'sqlite3', - database_path, + sqlite_restore_command = tuple( + shlex.quote(part) + for part in shlex.split(data_source.get('sqlite_restore_command') or 'sqlite3') ) - + restore_command = sqlite_restore_command + (shlex.quote(database_path),) # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning # if the restore paths don't exist in the archive. execute_command_with_processes( diff --git a/borgmatic/hooks/data_source/zfs.py b/borgmatic/hooks/data_source/zfs.py index 82a71e7a..9cb77587 100644 --- a/borgmatic/hooks/data_source/zfs.py +++ b/borgmatic/hooks/data_source/zfs.py @@ -1,5 +1,6 @@ import collections import glob +import hashlib import logging import os import shutil @@ -38,6 +39,9 @@ def get_datasets_to_backup(zfs_command, patterns): pattern paths represent the requested datasets to snapshot. But also include any datasets tagged with a borgmatic-specific user property, whether or not they appear in the patterns. + Only include datasets that contain at least one root pattern sourced from borgmatic + configuration (as opposed to generated elsewhere in borgmatic). + Return the result as a sequence of Dataset instances, sorted by mount point. ''' list_output = borgmatic.execute.execute_command_and_capture_output( @@ -48,7 +52,7 @@ def get_datasets_to_backup(zfs_command, patterns): '-t', 'filesystem', '-o', - f'name,mountpoint,{BORGMATIC_USER_PROPERTY}', + f'name,mountpoint,canmount,{BORGMATIC_USER_PROPERTY}', ) ) @@ -60,7 +64,12 @@ def get_datasets_to_backup(zfs_command, patterns): ( Dataset(dataset_name, mount_point, (user_property_value == 'auto'), ()) for line in list_output.splitlines() - for (dataset_name, mount_point, user_property_value) in (line.rstrip().split('\t'),) + for (dataset_name, mount_point, can_mount, user_property_value) in ( + line.rstrip().split('\t'), + ) + # Skip datasets that are marked "canmount=off", because mounting their snapshots will + # result in completely empty mount points—thereby preventing us from backing them up. + if can_mount == 'on' ), key=lambda dataset: dataset.mount_point, reverse=True, @@ -83,7 +92,12 @@ def get_datasets_to_backup(zfs_command, patterns): for contained_patterns in ( ( ( - (borgmatic.borg.pattern.Pattern(dataset.mount_point),) + ( + borgmatic.borg.pattern.Pattern( + dataset.mount_point, + source=borgmatic.borg.pattern.Pattern_source.HOOK, + ), + ) if dataset.auto_backup else () ) @@ -92,7 +106,12 @@ def get_datasets_to_backup(zfs_command, patterns): ) ), ) - if contained_patterns + if dataset.auto_backup + or any( + pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT + and pattern.source == borgmatic.borg.pattern.Pattern_source.CONFIG + for pattern in contained_patterns + ) ), key=lambda dataset: dataset.mount_point, ) @@ -115,7 +134,16 @@ def get_all_dataset_mount_points(zfs_command): ) ) - return tuple(sorted(line.rstrip() for line in list_output.splitlines())) + return tuple( + sorted( + { + mount_point + for line in list_output.splitlines() + for mount_point in (line.rstrip(),) + if mount_point != 'none' + } + ) + ) def snapshot_dataset(zfs_command, full_snapshot_name): # pragma: no cover @@ -155,10 +183,14 @@ def mount_snapshot(mount_command, full_snapshot_name, snapshot_mount_path): # p ) -def make_borg_snapshot_pattern(pattern, normalized_runtime_directory): +MOUNT_POINT_HASH_LENGTH = 10 + + +def make_borg_snapshot_pattern(pattern, dataset, normalized_runtime_directory): ''' - Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance, return a new Pattern with its - path rewritten to be in a snapshot directory based on the given runtime directory. + Given a Borg pattern as a borgmatic.borg.pattern.Pattern instance and the Dataset containing it, + return a new Pattern with its path rewritten to be in a snapshot directory based on both the + given runtime directory and the given Dataset's mount point. Move any initial caret in a regular expression pattern path to the beginning, so as not to break the regular expression. @@ -173,6 +205,10 @@ def make_borg_snapshot_pattern(pattern, normalized_runtime_directory): rewritten_path = initial_caret + os.path.join( normalized_runtime_directory, 'zfs_snapshots', + # Including this hash prevents conflicts between snapshot patterns for different datasets. + # For instance, without this, snapshotting a dataset at /var and another at /var/spool would + # result in overlapping snapshot patterns and therefore colliding mount attempts. + hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest(MOUNT_POINT_HASH_LENGTH), '.', # Borg 1.4+ "slashdot" hack. # Included so that the source directory ends up in the Borg archive at its "original" path. pattern.path.lstrip('^').lstrip(os.path.sep), @@ -183,6 +219,7 @@ def make_borg_snapshot_pattern(pattern, normalized_runtime_directory): pattern.type, pattern.style, pattern.device, + source=borgmatic.borg.pattern.Pattern_source.HOOK, ) @@ -209,7 +246,8 @@ def dump_data_sources( dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' logger.info(f'Snapshotting ZFS datasets{dry_run_label}') - # List ZFS datasets to get their mount points. + # List ZFS datasets to get their mount points, but only consider those patterns that came from + # actual user configuration (as opposed to, say, other hooks). zfs_command = hook_config.get('zfs_command', 'zfs') requested_datasets = get_datasets_to_backup(zfs_command, patterns) @@ -234,6 +272,9 @@ def dump_data_sources( snapshot_mount_path = os.path.join( normalized_runtime_directory, 'zfs_snapshots', + hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest( + MOUNT_POINT_HASH_LENGTH + ), dataset.mount_point.lstrip(os.path.sep), ) @@ -249,7 +290,9 @@ def dump_data_sources( ) for pattern in dataset.contained_patterns: - snapshot_pattern = make_borg_snapshot_pattern(pattern, normalized_runtime_directory) + snapshot_pattern = make_borg_snapshot_pattern( + pattern, dataset, normalized_runtime_directory + ) # Attempt to update the pattern in place, since pattern order matters to Borg. try: @@ -334,6 +377,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d os.path.normpath(borgmatic_runtime_directory), ), 'zfs_snapshots', + '*', ) logger.debug(f'Looking for snapshots to remove in {snapshots_glob}{dry_run_label}') umount_command = hook_config.get('umount_command', 'umount') @@ -346,7 +390,10 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d # child datasets before the shorter mount point paths of parent datasets. for mount_point in reversed(dataset_mount_points): snapshot_mount_path = os.path.join(snapshots_directory, mount_point.lstrip(os.path.sep)) - if not os.path.isdir(snapshot_mount_path): + + # If the snapshot mount path is empty, this is probably just a "shadow" of a nested + # dataset and therefore there's nothing to unmount. + if not os.path.isdir(snapshot_mount_path) or not os.listdir(snapshot_mount_path): continue # This might fail if the path is already mounted, but we swallow errors here since we'll @@ -370,10 +417,10 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d return except subprocess.CalledProcessError as error: logger.debug(error) - return + continue if not dry_run: - shutil.rmtree(snapshots_directory) + shutil.rmtree(snapshot_mount_path, ignore_errors=True) # Destroy snapshots. full_snapshot_names = get_all_snapshots(zfs_command) diff --git a/borgmatic/hooks/dispatch.py b/borgmatic/hooks/dispatch.py index 26b0832c..e12e70a0 100644 --- a/borgmatic/hooks/dispatch.py +++ b/borgmatic/hooks/dispatch.py @@ -3,6 +3,7 @@ import importlib import logging import pkgutil +import borgmatic.hooks.command import borgmatic.hooks.credential import borgmatic.hooks.data_source import borgmatic.hooks.monitoring diff --git a/borgmatic/hooks/monitoring/cronhub.py b/borgmatic/hooks/monitoring/cronhub.py index 837d770b..8d64957d 100644 --- a/borgmatic/hooks/monitoring/cronhub.py +++ b/borgmatic/hooks/monitoring/cronhub.py @@ -28,7 +28,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev filename in any log entries. If this is a dry run, then don't actually ping anything. ''' if state not in MONITOR_STATE_TO_CRONHUB: - logger.debug(f'Ignoring unsupported monitoring {state.name.lower()} in Cronhub hook') + logger.debug(f'Ignoring unsupported monitoring state {state.name.lower()} in Cronhub hook') return dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' diff --git a/borgmatic/hooks/monitoring/cronitor.py b/borgmatic/hooks/monitoring/cronitor.py index e8169312..2a176dc1 100644 --- a/borgmatic/hooks/monitoring/cronitor.py +++ b/borgmatic/hooks/monitoring/cronitor.py @@ -28,7 +28,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev filename in any log entries. If this is a dry run, then don't actually ping anything. ''' if state not in MONITOR_STATE_TO_CRONITOR: - logger.debug(f'Ignoring unsupported monitoring {state.name.lower()} in Cronitor hook') + logger.debug(f'Ignoring unsupported monitoring state {state.name.lower()} in Cronitor hook') return dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' diff --git a/borgmatic/hooks/monitoring/logs.py b/borgmatic/hooks/monitoring/logs.py index de1bef60..a2e2f39f 100644 --- a/borgmatic/hooks/monitoring/logs.py +++ b/borgmatic/hooks/monitoring/logs.py @@ -64,7 +64,7 @@ def get_handler(identifier): def format_buffered_logs_for_payload(identifier): ''' Get the handler previously added to the root logger, and slurp buffered logs out of it to - send to Healthchecks. + send to the monitoring service. ''' try: buffering_handler = get_handler(identifier) diff --git a/borgmatic/hooks/monitoring/ntfy.py b/borgmatic/hooks/monitoring/ntfy.py index f56a6047..9c479cdd 100644 --- a/borgmatic/hooks/monitoring/ntfy.py +++ b/borgmatic/hooks/monitoring/ntfy.py @@ -2,6 +2,8 @@ import logging import requests +import borgmatic.hooks.credential.parse + logger = logging.getLogger(__name__) @@ -47,9 +49,20 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev 'X-Tags': state_config.get('tags'), } - username = hook_config.get('username') - password = hook_config.get('password') - access_token = hook_config.get('access_token') + try: + username = borgmatic.hooks.credential.parse.resolve_credential( + hook_config.get('username'), config + ) + password = borgmatic.hooks.credential.parse.resolve_credential( + hook_config.get('password'), config + ) + access_token = borgmatic.hooks.credential.parse.resolve_credential( + hook_config.get('access_token'), config + ) + except ValueError as error: + logger.warning(f'Ntfy credential error: {error}') + return + auth = None if access_token is not None: diff --git a/borgmatic/hooks/monitoring/pagerduty.py b/borgmatic/hooks/monitoring/pagerduty.py index 6afc27c2..cd63f633 100644 --- a/borgmatic/hooks/monitoring/pagerduty.py +++ b/borgmatic/hooks/monitoring/pagerduty.py @@ -5,20 +5,37 @@ import platform import requests +import borgmatic.hooks.credential.parse +import borgmatic.hooks.monitoring.logs from borgmatic.hooks.monitoring import monitor logger = logging.getLogger(__name__) EVENTS_API_URL = 'https://events.pagerduty.com/v2/enqueue' +DEFAULT_LOGS_PAYLOAD_LIMIT_BYTES = 10000 +HANDLER_IDENTIFIER = 'pagerduty' -def initialize_monitor( - integration_key, config, config_filename, monitoring_log_level, dry_run -): # pragma: no cover +def initialize_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run): ''' - No initialization is necessary for this monitor. + Add a handler to the root logger that stores in memory the most recent logs emitted. That way, + we can send them all to PagerDuty upon a failure state. But skip this if the "send_logs" option + is false. ''' - pass + if hook_config.get('send_logs') is False: + return + + ping_body_limit = max( + DEFAULT_LOGS_PAYLOAD_LIMIT_BYTES + - len(borgmatic.hooks.monitoring.logs.PAYLOAD_TRUNCATION_INDICATOR), + 0, + ) + + borgmatic.hooks.monitoring.logs.add_handler( + borgmatic.hooks.monitoring.logs.Forgetful_buffering_handler( + HANDLER_IDENTIFIER, ping_body_limit, monitoring_log_level + ) + ) def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run): @@ -29,21 +46,30 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev ''' if state != monitor.State.FAIL: logger.debug( - f'Ignoring unsupported monitoring {state.name.lower()} in PagerDuty hook', + f'Ignoring unsupported monitoring state {state.name.lower()} in PagerDuty hook', ) return dry_run_label = ' (dry run; not actually sending)' if dry_run else '' logger.info(f'Sending failure event to PagerDuty {dry_run_label}') - if dry_run: + try: + integration_key = borgmatic.hooks.credential.parse.resolve_credential( + hook_config.get('integration_key'), config + ) + except ValueError as error: + logger.warning(f'PagerDuty credential error: {error}') return + logs_payload = borgmatic.hooks.monitoring.logs.format_buffered_logs_for_payload( + HANDLER_IDENTIFIER + ) + hostname = platform.node() local_timestamp = datetime.datetime.now(datetime.timezone.utc).astimezone().isoformat() payload = json.dumps( { - 'routing_key': hook_config['integration_key'], + 'routing_key': integration_key, 'event_action': 'trigger', 'payload': { 'summary': f'backup failed on {hostname}', @@ -57,11 +83,14 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev 'hostname': hostname, 'configuration filename': config_filename, 'server time': local_timestamp, + 'logs': logs_payload, }, }, } ) - logger.debug(f'Using PagerDuty payload: {payload}') + + if dry_run: + return logging.getLogger('urllib3').setLevel(logging.ERROR) try: @@ -74,6 +103,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev def destroy_monitor(ping_url_or_uuid, config, monitoring_log_level, dry_run): # pragma: no cover ''' - No destruction is necessary for this monitor. + Remove the monitor handler that was added to the root logger. This prevents the handler from + getting reused by other instances of this monitor. ''' - pass + borgmatic.hooks.monitoring.logs.remove_handler(HANDLER_IDENTIFIER) diff --git a/borgmatic/hooks/monitoring/pushover.py b/borgmatic/hooks/monitoring/pushover.py index 8211c32d..86a1ac26 100644 --- a/borgmatic/hooks/monitoring/pushover.py +++ b/borgmatic/hooks/monitoring/pushover.py @@ -2,6 +2,8 @@ import logging import requests +import borgmatic.hooks.credential.parse + logger = logging.getLogger(__name__) @@ -32,8 +34,14 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev state_config = hook_config.get(state.name.lower(), {}) - token = hook_config.get('token') - user = hook_config.get('user') + try: + token = borgmatic.hooks.credential.parse.resolve_credential( + hook_config.get('token'), config + ) + user = borgmatic.hooks.credential.parse.resolve_credential(hook_config.get('user'), config) + except ValueError as error: + logger.warning(f'Pushover credential error: {error}') + return logger.info(f'Updating Pushover{dry_run_label}') diff --git a/borgmatic/hooks/monitoring/uptime_kuma.py b/borgmatic/hooks/monitoring/uptime_kuma.py index 015dde0b..68d5f29b 100644 --- a/borgmatic/hooks/monitoring/uptime_kuma.py +++ b/borgmatic/hooks/monitoring/uptime_kuma.py @@ -37,7 +37,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev logging.getLogger('urllib3').setLevel(logging.ERROR) try: - response = requests.get(f'{push_url}?{query}') + response = requests.get(f'{push_url}?{query}', verify=hook_config.get('verify_tls', True)) if not response.ok: response.raise_for_status() except requests.exceptions.RequestException as error: diff --git a/borgmatic/hooks/monitoring/zabbix.py b/borgmatic/hooks/monitoring/zabbix.py index 96b0a9da..9235147f 100644 --- a/borgmatic/hooks/monitoring/zabbix.py +++ b/borgmatic/hooks/monitoring/zabbix.py @@ -2,6 +2,8 @@ import logging import requests +import borgmatic.hooks.credential.parse + logger = logging.getLogger(__name__) @@ -14,6 +16,42 @@ def initialize_monitor( pass +def send_zabbix_request(server, headers, data): + ''' + Given a Zabbix server URL, HTTP headers as a dict, and valid Zabbix JSON payload data as a dict, + send a request to the Zabbix server via API. + + Return the response "result" value or None. + ''' + logging.getLogger('urllib3').setLevel(logging.ERROR) + + logger.debug(f'Sending a "{data["method"]}" request to the Zabbix server') + + try: + response = requests.post(server, headers=headers, json=data) + + if not response.ok: + response.raise_for_status() + except requests.exceptions.RequestException as error: + logger.warning(f'Zabbix error: {error}') + + return None + + try: + result = response.json().get('result') + error_message = result['data'][0]['error'] + except requests.exceptions.JSONDecodeError: + logger.warning('Zabbix error: Cannot parse API response') + + return None + except (TypeError, KeyError, IndexError): + return result + else: + logger.warning(f'Zabbix error: {error_message}') + + return None + + def ping_monitor(hook_config, config, config_filename, state, monitoring_log_level, dry_run): ''' Update the configured Zabbix item using either the itemid, or a host and key. @@ -34,23 +72,31 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev }, ) + try: + username = borgmatic.hooks.credential.parse.resolve_credential( + hook_config.get('username'), config + ) + password = borgmatic.hooks.credential.parse.resolve_credential( + hook_config.get('password'), config + ) + api_key = borgmatic.hooks.credential.parse.resolve_credential( + hook_config.get('api_key'), config + ) + except ValueError as error: + logger.warning(f'Zabbix credential error: {error}') + + return + server = hook_config.get('server') - username = hook_config.get('username') - password = hook_config.get('password') - api_key = hook_config.get('api_key') itemid = hook_config.get('itemid') host = hook_config.get('host') key = hook_config.get('key') value = state_config.get('value') headers = {'Content-Type': 'application/json-rpc'} - logger.info(f'Updating Zabbix{dry_run_label}') + logger.info(f'Pinging Zabbix{dry_run_label}') logger.debug(f'Using Zabbix URL: {server}') - if server is None: - logger.warning('Server missing for Zabbix') - return - # Determine the Zabbix method used to store the value: itemid or host/key if itemid is not None: logger.info(f'Updating {itemid} on Zabbix') @@ -61,8 +107,8 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev 'id': 1, } - elif (host and key) is not None: - logger.info(f'Updating Host:{host} and Key:{key} on Zabbix') + elif host is not None and key is not None: + logger.info(f'Updating Host: "{host}" and Key: "{key}" on Zabbix') data = { 'jsonrpc': '2.0', 'method': 'history.push', @@ -72,58 +118,63 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev elif host is not None: logger.warning('Key missing for Zabbix') - return + return elif key is not None: logger.warning('Host missing for Zabbix') + return else: logger.warning('No Zabbix itemid or host/key provided') + return # Determine the authentication method: API key or username/password if api_key is not None: logger.info('Using API key auth for Zabbix') - headers['Authorization'] = 'Bearer ' + api_key - - elif (username and password) is not None: - logger.info('Using user/pass auth with user {username} for Zabbix') - auth_data = { + headers['Authorization'] = f'Bearer {api_key}' + elif username is not None and password is not None: + logger.info(f'Using user/pass auth with user {username} for Zabbix') + login_data = { 'jsonrpc': '2.0', 'method': 'user.login', 'params': {'username': username, 'password': password}, 'id': 1, } + if not dry_run: - logging.getLogger('urllib3').setLevel(logging.ERROR) - try: - response = requests.post(server, headers=headers, json=auth_data) - data['auth'] = response.json().get('result') - if not response.ok: - response.raise_for_status() - except requests.exceptions.RequestException as error: - logger.warning(f'Zabbix error: {error}') + result = send_zabbix_request(server, headers, login_data) + + if not result: return + headers['Authorization'] = f'Bearer {result}' elif username is not None: logger.warning('Password missing for Zabbix authentication') - return + return elif password is not None: logger.warning('Username missing for Zabbix authentication') + return else: logger.warning('Authentication data missing for Zabbix') + return if not dry_run: - logging.getLogger('urllib3').setLevel(logging.ERROR) - try: - response = requests.post(server, headers=headers, json=data) - if not response.ok: - response.raise_for_status() - except requests.exceptions.RequestException as error: - logger.warning(f'Zabbix error: {error}') + send_zabbix_request(server, headers, data) + + if username is not None and password is not None: + logout_data = { + 'jsonrpc': '2.0', + 'method': 'user.logout', + 'params': [], + 'id': 1, + } + + if not dry_run: + send_zabbix_request(server, headers, logout_data) def destroy_monitor(ping_url_or_uuid, config, monitoring_log_level, dry_run): # pragma: no cover diff --git a/borgmatic/logger.py b/borgmatic/logger.py index 0ece32ba..4eb34a34 100644 --- a/borgmatic/logger.py +++ b/borgmatic/logger.py @@ -29,12 +29,13 @@ def interactive_console(): return sys.stderr.isatty() and os.environ.get('TERM') != 'dumb' -def should_do_markup(no_color, configs): +def should_do_markup(configs, json_enabled): ''' - Given the value of the command-line no-color argument, and a dict of configuration filename to - corresponding parsed configuration, determine if we should enable color marking up. + Given a dict of configuration filename to corresponding parsed configuration (which already have + any command-line overrides applied) and whether json is enabled, determine if we should enable + color marking up. ''' - if no_color: + if json_enabled: return False if any(config.get('color', True) is False for config in configs.values()): @@ -256,7 +257,7 @@ class Log_prefix: self.original_prefix = get_log_prefix() set_log_prefix(self.prefix) - def __exit__(self, exception, value, traceback): + def __exit__(self, exception_type, exception, traceback): ''' Restore any original prefix. ''' @@ -265,9 +266,14 @@ class Log_prefix: class Delayed_logging_handler(logging.handlers.BufferingHandler): ''' - A logging handler that buffers logs and doesn't flush them until explicitly flushed (after target - handlers are actually set). It's useful for holding onto logged records from before logging is - configured to ensure those records eventually make their way to the relevant logging handlers. + A logging handler that buffers logs and doesn't flush them until explicitly flushed (after + target handlers are actually set). It's useful for holding onto messages logged before logging + is configured, ensuring those records eventually make their way to the relevant logging + handlers. + + When flushing, don't forward log records to a target handler if the record's log level is below + that of the handler. This recreates the standard logging behavior of, say, logging.DEBUG records + getting suppressed if a handler's level is only set to logging.INFO. ''' def __init__(self): @@ -287,7 +293,8 @@ class Delayed_logging_handler(logging.handlers.BufferingHandler): for record in self.buffer: for target in self.targets: - target.handle(record) + if record.levelno >= target.level: + target.handle(record) self.buffer.clear() finally: diff --git a/borgmatic/signals.py b/borgmatic/signals.py index 7ec31d91..40fa5295 100644 --- a/borgmatic/signals.py +++ b/borgmatic/signals.py @@ -24,6 +24,9 @@ def handle_signal(signal_number, frame): logger.critical('Exiting due to TERM signal') sys.exit(EXIT_CODE_FROM_SIGNAL + signal.SIGTERM) elif signal_number == signal.SIGINT: + # Borg doesn't always exit on a SIGINT, so give it a little encouragement. + os.killpg(os.getpgrp(), signal.SIGTERM) + raise KeyboardInterrupt() diff --git a/docs/Dockerfile b/docs/Dockerfile index c3279570..cfa1b828 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -4,7 +4,7 @@ COPY . /app RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib RUN pip install --break-system-packages --no-cache /app && borgmatic config generate && chmod +r /etc/borgmatic/config.yaml RUN borgmatic --help > /command-line.txt \ - && for action in repo-create transfer create prune compact check delete extract config "config bootstrap" "config generate" "config validate" export-tar mount umount repo-delete restore repo-list list repo-info info break-lock "key export" "key change-passphrase" borg; do \ + && for action in repo-create transfer create prune compact check delete extract config "config bootstrap" "config generate" "config validate" export-tar mount umount repo-delete restore repo-list list repo-info info break-lock "key export" "key import" "key change-passphrase" recreate borg; do \ echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \ && borgmatic $action --help >> /command-line.txt; done RUN /app/docs/fetch-contributors >> /contributors.html diff --git a/docs/_includes/index.css b/docs/_includes/index.css index 56841ecb..c798f2e9 100644 --- a/docs/_includes/index.css +++ b/docs/_includes/index.css @@ -165,6 +165,7 @@ ul { } li { padding: .25em 0; + line-height: 1.5; } li ul { list-style-type: disc; diff --git a/docs/_includes/layouts/base.njk b/docs/_includes/layouts/base.njk index 1119e27b..0807f5f0 100644 --- a/docs/_includes/layouts/base.njk +++ b/docs/_includes/layouts/base.njk @@ -4,7 +4,7 @@ - + {{ subtitle + ' - ' if subtitle}}{{ title }} {%- set css %} {% include 'index.css' %} diff --git a/docs/fetch-contributors b/docs/fetch-contributors index 81ee6e22..7c1e1093 100755 --- a/docs/fetch-contributors +++ b/docs/fetch-contributors @@ -26,8 +26,7 @@ def list_merged_pulls(url): def list_contributing_issues(url): - # labels = bug, design finalized, etc. - response = requests.get(f'{url}?labels=19,20,22,23,32,52,53,54', headers={'Accept': 'application/json', 'Content-Type': 'application/json'}) + response = requests.get(url, headers={'Accept': 'application/json', 'Content-Type': 'application/json'}) if not response.ok: response.raise_for_status() @@ -39,7 +38,7 @@ PULLS_API_ENDPOINT_URLS = ( 'https://projects.torsion.org/api/v1/repos/borgmatic-collective/borgmatic/pulls', 'https://api.github.com/repos/borgmatic-collective/borgmatic/pulls', ) -ISSUES_API_ENDPOINT_URL = 'https://projects.torsion.org/api/v1/repos/borgmatic-collective/borgmatic/issues' +ISSUES_API_ENDPOINT_URL = 'https://projects.torsion.org/api/v1/repos/borgmatic-collective/borgmatic/issues?state=all' RECENT_CONTRIBUTORS_CUTOFF_DAYS = 365 diff --git a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md index 02e41f43..3aecf478 100644 --- a/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md +++ b/docs/how-to/add-preparation-and-cleanup-steps-to-backups.md @@ -7,18 +7,112 @@ eleventyNavigation: --- ## Preparation and cleanup hooks -If you find yourself performing preparation tasks before your backup runs, or -cleanup work afterwards, borgmatic hooks may be of interest. Hooks are shell -commands that borgmatic executes for you at various points as it runs, and -they're configured in the `hooks` section of your configuration file. But if -you're looking to backup a database, it's probably easier to use the [database -backup -feature](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/) -instead. +If you find yourself performing preparation tasks before your backup runs or +doing cleanup work afterwards, borgmatic command hooks may be of interest. These +are custom shell commands you can configure borgmatic to execute at various +points as it runs. -You can specify `before_backup` hooks to perform preparation steps before +(But if you're looking to backup a database, it's probably easier to use the +[database backup +feature](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/) +instead.) + +New in version 2.0.0 (**not yet +released**) Command hooks are now configured via a list of `commands:` in +your borgmatic configuration file. For example: + +```yaml +commands: + - before: action + when: [create] + run: + - echo "Before create!" + - after: action + when: + - create + - prune + run: + - echo "After create or prune!" + - after: error + run: + - echo "Something went wrong!" +``` + +If you're coming from an older version of borgmatic, there is tooling to help +you [upgrade your +configuration](https://torsion.org/borgmatic/docs/how-to/upgrade/#upgrading-your-configuration) +to this new command hook format. + +Note that if a `run:` command contains a special YAML character such as a colon, +you may need to quote the entire string (or use a [multiline +string](https://yaml-multiline.info/)) to avoid an error: + +```yaml +commands: + - before: action + when: [create] + run: + - "echo Backup: start" +``` + +Each command in the `commands:` list has the following options: + + * `before` or `after`: Name for the point in borgmatic's execution that the commands should be run before or after, one of: + * `action` runs before each action for each repository. This replaces the deprecated `before_create`, `after_prune`, etc. + * `repository` runs before or after all actions for each repository. This replaces the deprecated `before_actions` and `after_actions`. + * `configuration` runs before or after all actions and repositories in the current configuration file. + * `everything` runs before or after all configuration files. Errors here do not trigger `error` hooks or the `fail` state in monitoring hooks. This replaces the deprecated `before_everything` and `after_everything`. + * `error` runs after an error occurs—and it's only available for `after`. This replaces the deprecated `on_error` hook. + * `when`: Only trigger the hook when borgmatic is run with particular actions (`create`, `prune`, etc.) listed here. Defaults to running for all actions. + * `run`: List of one or more shell commands or scripts to run when this command hook is triggered. + +An `after` command hook runs even if an error occurs in the corresponding +`before` hook or between those two hooks. This allows you to perform cleanup +steps that correspond to `before` preparation commands—even when something goes +wrong. This is a departure from the way that the deprecated `after_*` hooks +worked in borgmatic prior to version 2.0.0. + +Additionally, when command hooks run, they respect the `working_directory` +option if it is configured, meaning that the hook commands are run in that +directory. + + +### Order of execution + +Here's a way of visualizing how all of these command hooks slot into borgmatic's +execution. + +Let's say you've got a borgmatic configuration file with a configured +repository. And suppose you configure several command hooks and then run +borgmatic for the `create` and `prune` actions. Here's the order of execution: + + * Run `before: everything` hooks (from all configuration files). + * Run `before: configuration` hooks (from the first configuration file). + * Run `before: repository` hooks (for the first repository). + * Run `before: action` hooks for `create`. + * Actually run the `create` action (e.g. `borg create`). + * Run `after: action` hooks for `create`. + * Run `before: action` hooks for `prune`. + * Actually run the `prune` action (e.g. `borg prune`). + * Run `after: action` hooks for `prune`. + * Run `after: repository` hooks (for the first repository). + * Run `after: configuration` hooks (from the first configuration file). + * Run `after: everything` hooks (from all configuration files). + +This same order of execution extends to multiple repositories and/or +configuration files. + + +### Deprecated command hooks + +Prior to version 2.0.0 The +command hooks worked a little differently. In these older versions of borgmatic, +you can specify `before_backup` hooks to perform preparation steps before running backups and specify `after_backup` hooks to perform cleanup steps -afterwards. Here's an example: +afterwards. These deprecated command hooks still work in version 2.0.0+, +although see below about a few semantic differences starting in that version. + +Here's an example of these deprecated hooks: ```yaml before_backup: @@ -43,6 +137,15 @@ instance, `before_prune` runs before a `prune` action for a repository, while Prior to version 1.8.0 Put these options in the `hooks:` section of your configuration. +New in version 2.0.0 An `after_*` +command hook runs even if an error occurs in the corresponding `before_*` hook +or between those two hooks. This allows you to perform cleanup steps that +correspond to `before_*` preparation commands—even when something goes wrong. + +New in version 2.0.0 When command +hooks run, they respect the `working_directory` option if it is configured, +meaning that the hook commands are run in that directory. + New in version 1.7.0 The `before_actions` and `after_actions` hooks run before/after all the actions (like `create`, `prune`, etc.) for each repository. These hooks are a good @@ -57,49 +160,13 @@ but not if an error occurs in a previous hook or in the backups themselves. (Prior to borgmatic 1.6.0, these hooks instead ran once per configuration file rather than once per repository.) - -## Variable interpolation - -The before and after action hooks support interpolating particular runtime -variables into the hook command. Here's an example that assumes you provide a -separate shell script: - -```yaml -after_prune: - - record-prune.sh "{configuration_filename}" "{repository}" -``` - -Prior to version 1.8.0 Put -this option in the `hooks:` section of your configuration. - -In this example, when the hook is triggered, borgmatic interpolates runtime -values into the hook command: the borgmatic configuration filename and the -paths of the current Borg repository. Here's the full set of supported -variables you can use here: - - * `configuration_filename`: borgmatic configuration filename in which the - hook was defined - * `log_file` - New in version 1.7.12: - path of the borgmatic log file, only set when the `--log-file` flag is used - * `repository`: path of the current repository as configured in the current - borgmatic configuration file - * `repository_label` New in version - 1.8.12: label of the current repository as configured in the current - borgmatic configuration file - -Note that you can also interpolate in [arbitrary environment -variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/). - - -## Global hooks - You can also use `before_everything` and `after_everything` hooks to perform global setup or cleanup: ```yaml before_everything: - set-up-stuff-globally + after_everything: - clean-up-stuff-globally ``` @@ -117,13 +184,102 @@ but only if there is a `create` action. It runs even if an error occurs during a backup or a backup hook, but not if an error occurs during a `before_everything` hook. +`on_error` hooks run when an error occurs, but only if there is a `create`, +`prune`, `compact`, or `check` action. For instance, borgmatic can run +configurable shell commands to fire off custom error notifications or take other +actions, so you can get alerted as soon as something goes wrong. Here's a +not-so-useful example: -## Error hooks +```yaml +on_error: + - echo "Error while creating a backup or running a backup hook." +``` -borgmatic also runs `on_error` hooks if an error occurs, either when creating -a backup or running a backup hook. See the [monitoring and alerting -documentation](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/) -for more information. +Prior to version 1.8.0 Put +this option in the `hooks:` section of your configuration. + +The `on_error` hook supports interpolating particular runtime variables into +the hook command. Here's an example that assumes you provide a separate shell +script to handle the alerting: + +```yaml +on_error: + - send-text-message.sh +``` + +borgmatic does not run `on_error` hooks if an error occurs within a +`before_everything` or `after_everything` hook. + + +## Variable interpolation + +The command action hooks support interpolating particular runtime variables into +the commands that are run. Here's are a couple examples that assume you provide +separate shell scripts: + +```yaml +commands: + - after: action + when: [prune] + run: + - record-prune.sh {configuration_filename} {repository} + - after: error + when: [create] + run: + - send-text-message.sh {configuration_filename} {repository} +``` + +In this example, when the hook is triggered, borgmatic interpolates runtime +values into each hook command: the borgmatic configuration filename and the +paths of the current Borg repository. + +Here's the full set of supported variables you can use here: + + * `configuration_filename`: borgmatic configuration filename in which the + hook was defined + * `log_file` + New in version 1.7.12: + path of the borgmatic log file, only set when the `--log-file` flag is used + * `repository`: path of the current repository as configured in the current + borgmatic configuration file, if applicable to the current hook + * `repository_label` New in version + 1.8.12: label of the current repository as configured in the current + borgmatic configuration file, if applicable to the current hook + * `error`: the error message itself, only applies to `error` hooks + * `output`: output of the command that failed, only applies to `error` hooks + (may be blank if an error occurred without running a command) + +Not all command hooks support all variables. For instance, the `everything` and +`configuration` hooks don't support repository variables because those hooks +don't run in the context of a single repository. But the deprecated command +hooks (`before_backup`, `on_error`, etc.) do generally support variable +interpolation. + +borgmatic automatically escapes these interpolated values to prevent shell +injection attacks. One implication is that you shouldn't wrap the interpolated +values in your own quotes, as that will interfere with the quoting performed by +borgmatic and result in your command receiving incorrect arguments. For +instance, this won't work: + +```yaml +commands: + - after: error + run: + # Don't do this! It won't work, as the {error} value is already quoted. + - send-text-message.sh "Uh oh: {error}" +``` + +Do this instead: + +```yaml +commands: + - after: error + run: + - send-text-message.sh {error} +``` + +Note that you can also interpolate [arbitrary environment +variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/). ## Hook output diff --git a/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md b/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md index d54d18f5..4517ee8b 100644 --- a/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md +++ b/docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md @@ -29,17 +29,14 @@ concept of "soft failure" come in. This feature leverages [borgmatic command hooks](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/), -so first familiarize yourself with them. The idea is that you write a simple -test in the form of a borgmatic hook to see if backups should proceed or not. +so familiarize yourself with them first. The idea is that you write a simple +test in the form of a borgmatic command hook to see if backups should proceed or +not. The way the test works is that if any of your hook commands return a special exit status of 75, that indicates to borgmatic that it's a temporary failure, and borgmatic should skip all subsequent actions for the current repository. -Prior to version 1.9.0 Soft -failures skipped subsequent actions for *all* repositories in the -configuration file, rather than just for the current repository. - If you return any status besides 75, then it's a standard success or error. (Zero is success; anything else other than 75 is an error). @@ -62,33 +59,37 @@ these options in the `location:` section of your configuration. Prior to version 1.7.10 Omit the `path:` portion of the `repositories` list. -Then, write a `before_backup` hook in that same configuration file that uses -the external `findmnt` utility to see whether the drive is mounted before -proceeding. +Then, make a command hook in that same configuration file that uses the external +`findmnt` utility to see whether the drive is mounted before proceeding. ```yaml -before_backup: +commands: + - before: repository + run: + - findmnt /mnt/removable > /dev/null || exit 75 +``` + +Prior to version 2.0.0 Use the +deprecated `before_actions` hook instead: + +```yaml +before_actions: - findmnt /mnt/removable > /dev/null || exit 75 ``` Prior to version 1.8.0 Put this option in the `hooks:` section of your configuration. +Prior to version 1.7.0 Use +`before_create` or similar instead of `before_actions`, which was introduced in +borgmatic 1.7.0. + What this does is check if the `findmnt` command errors when probing for a particular mount point. If it does error, then it returns exit code 75 to borgmatic. borgmatic logs the soft failure, skips all further actions for the current repository, and proceeds onward to any other repositories and/or configuration files you may have. -If you'd prefer not to use a separate configuration file, and you'd rather -have multiple repositories in a single configuration file, you can make your -`before_backup` soft failure test [vary by -repository](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/#variable-interpolation). -That might require calling out to a separate script though. - -Note that `before_backup` only runs on the `create` action. See below about -optionally using `before_actions` instead. - You can imagine a similar check for the sometimes-online server case: ```yaml @@ -98,50 +99,50 @@ source_directories: repositories: - path: ssh://me@buddys-server.org/./backup.borg -before_backup: - - ping -q -c 1 buddys-server.org > /dev/null || exit 75 +commands: + - before: repository + run: + - ping -q -c 1 buddys-server.org > /dev/null || exit 75 ``` Or to only run backups if the battery level is high enough: ```yaml -before_backup: - - is_battery_percent_at_least.sh 25 +commands: + - before: repository + run: + - is_battery_percent_at_least.sh 25 ``` -(Writing the battery script is left as an exercise to the reader.) - -New in version 1.7.0 The -`before_actions` and `after_actions` hooks run before/after all the actions -(like `create`, `prune`, etc.) for each repository. So if you'd like your soft -failure command hook to run regardless of action, consider using -`before_actions` instead of `before_backup`. +Writing the battery script is left as an exercise to the reader. ## Caveats and details There are some caveats you should be aware of with this feature. - * You'll generally want to put a soft failure command in the `before_backup` + * You'll generally want to put a soft failure command in a `before` command hook, so as to gate whether the backup action occurs. While a soft failure is - also supported in the `after_backup` hook, returning a soft failure there + also supported in an `after` command hook, returning a soft failure there won't prevent any actions from occurring, because they've already occurred! - Similarly, you can return a soft failure from an `on_error` hook, but at + Similarly, you can return a soft failure from an `error` command hook, but at that point it's too late to prevent the error. * Returning a soft failure does prevent further commands in the same hook from - executing. So, like a standard error, it is an "early out". Unlike a standard + executing. So, like a standard error, it is an "early out." Unlike a standard error, borgmatic does not display it in angry red text or consider it a failure. - * Any given soft failure only applies to the a single borgmatic repository - (as of borgmatic 1.9.0). So if you have other repositories you don't want - soft-failed, then make your soft fail test [vary by - repository](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/#variable-interpolation)—or - put anything that you don't want soft-failed (like always-online cloud - backups) in separate configuration files from your soft-failing - repositories. + * New in version 1.9.0 Soft + failures in `action` or `before_*` command hooks only skip the current + repository rather than all repositories in a configuration file. + * If you're writing a soft failure script that you want to vary based on the + current repository, for instance so you can have multiple repositories in a + single configuration file, have a look at [command hook variable + interpolation](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/#variable-interpolation). + And there's always still the option of putting anything that you don't want + soft-failed (like always-online cloud backups) in separate configuration + files from your soft-failing repositories. * The soft failure doesn't have to test anything related to a repository. You - can even perform a test to make sure that individual source directories are - mounted and available. Use your imagination! - * The soft failure feature also works for before/after hooks for other - actions as well. But it is not implemented for `before_everything` or - `after_everything`. + can even perform a test that individual source directories are mounted and + available. Use your imagination! + * Soft failures are not currently implemented for `everything`, + `before_everything`, or `after_everything` command hooks. diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index 769d4dbf..d4acb967 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -193,14 +193,14 @@ mysql_databases: ### Containers -If your database is running within a container and borgmatic is too, no +If your database server is running within a container and borgmatic is too, no problem—configure borgmatic to connect to the container's name on its exposed port. For instance: ```yaml postgresql_databases: - name: users - hostname: your-database-container-name + hostname: your-database-server-container-name port: 5433 username: postgres password: trustsome1 @@ -210,21 +210,22 @@ postgresql_databases: these options in the `hooks:` section of your configuration. But what if borgmatic is running on the host? You can still connect to a -database container if its ports are properly exposed to the host. For +database server container if its ports are properly exposed to the host. For instance, when running the database container, you can specify `--publish 127.0.0.1:5433:5432` so that it exposes the container's port 5432 to port 5433 -on the host (only reachable on localhost, in this case). Or the same thing -with Docker Compose: +on the host (only reachable on localhost, in this case). Or the same thing with +Docker Compose: ```yaml services: - your-database-container-name: + your-database-server-container-name: image: postgres ports: - 127.0.0.1:5433:5432 ``` -And then you can connect to the database from borgmatic running on the host: +And then you can configure borgmatic running on the host to connect to the +database: ```yaml hooks: @@ -240,9 +241,9 @@ Alter the ports in these examples to suit your particular database system. Normally, borgmatic dumps a database by running a database dump command (e.g. `pg_dump`) on the host or wherever borgmatic is running, and this command -connects to your containerized database via the given `hostname` and `port`. -But if you don't have any database dump commands installed on your host and -you'd rather use the commands inside your database container itself, borgmatic +connects to your containerized database via the given `hostname` and `port`. But +if you don't have any database dump commands installed on your host and you'd +rather use the commands inside your running database container itself, borgmatic supports that too. For that, configure borgmatic to `exec` into your container to run the dump command. @@ -259,9 +260,10 @@ hooks: pg_dump_command: docker exec my_pg_container pg_dump ``` -... where `my_pg_container` is the name of your database container. In this -example, you'd also need to set the `pg_restore_command` and `psql_command` -options. +... where `my_pg_container` is the name of your running database container. +Running `pg_dump` this way takes advantage of the localhost "trust" +authentication within that container. In this example, you'd also need to set +the `pg_restore_command` and `psql_command` options. If you choose to use the `pg_dump` command within the container, and you're using the `directory` format in particular, you'll also need to mount the @@ -280,6 +282,24 @@ services: - /run/user/1000:/run/user/1000 ``` +Another variation: If you're running borgmatic on the host but want to spin up a +temporary `pg_dump` container whenever borgmatic dumps a database, for +instance to make use of a `pg_dump` version not present on the host, try +something like this: + +```yaml +hooks: + postgresql_databases: + - name: users + hostname: your-database-hostname + username: postgres + password: trustsome1 + pg_dump_command: docker run --rm --env PGPASSWORD postgres:17-alpine pg_dump +``` + +The `--env PGPASSWORD` is necessary here for borgmatic to provide your database +password to the temporary `pg_dump` container. + Similar command override options are available for (some of) the other supported database types as well. See the [configuration reference](https://torsion.org/borgmatic/docs/reference/configuration/) for @@ -309,10 +329,8 @@ hooks: ### External passwords If you don't want to keep your database passwords in your borgmatic -configuration file, you can instead pass them in via [environment -variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/) -or command-line [configuration -overrides](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides). +configuration file, you can instead pass them in [from external credential +sources](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/). ### Configuration backups diff --git a/docs/how-to/make-per-application-backups.md b/docs/how-to/make-per-application-backups.md index 095fef5f..a81cd943 100644 --- a/docs/how-to/make-per-application-backups.md +++ b/docs/how-to/make-per-application-backups.md @@ -482,16 +482,89 @@ applications, but then set the repository for each application at runtime. Or you might want to try a variant of an option for testing purposes without actually touching your configuration file. +New in version 2.0.0 Whatever the reason, you can override borgmatic configuration options at the -command-line via the `--override` flag. Here's an example: +command-line, as there's a command-line flag corresponding to every +configuration option (with its underscores converted to dashes). + +For instance, to override the `compression` configuration option, use the +corresponding `--compression` flag on the command-line: + +```bash +borgmatic create --compression zstd +``` + +What this does is load your given configuration files and for each one, disregard +the configured value for the `compression` option and use the value given on the +command-line instead—but just for the duration of the borgmatic run. + +You can override nested configuration options too by separating such option +names with a period. For instance: + +```bash +borgmatic create --bootstrap.store-config-files false +``` + +You can even set complex option data structures by using inline YAML syntax. For +example, set the `repositories` option with a YAML list of key/value pairs: + +```bash +borgmatic create --repositories "[{path: /mnt/backup, label: local}]" +``` + +If your override value contains characters like colons or spaces, then you'll +need to use quotes for it to parse correctly. + +You can also set individual nested options within existing list elements: + +```bash +borgmatic create --repositories[0].path /mnt/backup +``` + +This updates the `path` option for the first repository in `repositories`. +Change the `[0]` index as needed to address different list elements. And note +that this only works for elements already set in configuration; you can't append +new list elements from the command-line. + +See the [command-line reference +documentation](https://torsion.org/borgmatic/docs/reference/command-line/) for +the full set of available arguments, including examples of each for the complex +values. + +There are a handful of configuration options that don't have corresponding +command-line flags at the global scope, but instead have flags within individual +borgmatic actions. For instance, the `list_details` option can be overridden by +the `--list` flag that's only present on particular actions. Similarly with +`progress` and `--progress`, `statistics` and `--stats`, and `match_archives` +and `--match-archives`. + +Also note that if you want to pass a command-line flag itself as a value to one +of these override flags, that may not work. For instance, specifying +`--extra-borg-options.create --no-cache-sync` results in an error, because +`--no-cache-sync` gets interpreted as a borgmatic option (which in this case +doesn't exist) rather than a Borg option. + +An alternate to command-line overrides is passing in your values via +[environment +variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/). + + +### Deprecated overrides + +Prior to version 2.0.0 +Configuration overrides were performed with an `--override` flag. You can still +use `--override` with borgmatic 2.0.0+, but it's deprecated in favor of the new +command-line flags described above. + +Here's an example of `--override`: ```bash borgmatic create --override remote_path=/usr/local/bin/borg1 ``` -What this does is load your configuration files and for each one, disregard -the configured value for the `remote_path` option and use the value of -`/usr/local/bin/borg1` instead. +What this does is load your given configuration files and for each one, disregard +the configured value for the `remote_path` option and use the value given on the +command-line instead—but just for the duration of the borgmatic run. You can even override nested values or multiple values at once. For instance: @@ -540,10 +613,6 @@ reference](https://torsion.org/borgmatic/docs/reference/configuration/) for which options are list types. (YAML list values look like `- this` with an indentation and a leading dash.) -An alternate to command-line overrides is passing in your values via -[environment -variables](https://torsion.org/borgmatic/docs/how-to/provide-your-passwords/). - ## Constant interpolation diff --git a/docs/how-to/monitor-your-backups.md b/docs/how-to/monitor-your-backups.md index 43acdb08..9cdbd4be 100644 --- a/docs/how-to/monitor-your-backups.md +++ b/docs/how-to/monitor-your-backups.md @@ -14,140 +14,55 @@ and alerting comes in. There are several different ways you can monitor your backups and find out whether they're succeeding. Which of these you choose to do is up to you and -your particular infrastructure. +your particular infrastructure: -### Job runner alerts - -The easiest place to start is with failure alerts from the [scheduled job -runner](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#autopilot) -(cron, systemd, etc.) that's running borgmatic. But note that if the job -doesn't even get scheduled (e.g. due to the job runner not running), you -probably won't get an alert at all! Still, this is a decent first line of -defense, especially when combined with some of the other approaches below. - -### Commands run on error - -The `on_error` hook allows you to run an arbitrary command or script when -borgmatic itself encounters an error running your backups. So for instance, -you can run a script to send yourself a text message alert. But note that if -borgmatic doesn't actually run, this alert won't fire. See [error -hooks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hooks) -below for how to configure this. - -### Third-party monitoring services - -borgmatic integrates with these monitoring services and libraries, pinging -them as backups happen: - - * [Apprise](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#apprise-hook) - * [Cronhub](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook) - * [Cronitor](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook) - * [Grafana Loki](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#loki-hook) - * [Healthchecks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook) - * [ntfy](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#ntfy-hook) - * [PagerDuty](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook) - * [Pushover](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pushover-hook) - * [Sentry](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#sentry-hook) - * [Uptime Kuma](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#uptime-kuma-hook) - * [Zabbix](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#zabbix-hook) - -The idea is that you'll receive an alert when something goes wrong or when the -service doesn't hear from borgmatic for a configured interval (if supported). -See the documentation links above for configuration information. - -While these services and libraries offer different features, you probably only -need to use one of them at most. - - -### Third-party monitoring software - -You can use traditional monitoring software to consume borgmatic JSON output -and track when the last successful backup occurred. See [scripting -borgmatic](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#scripting-borgmatic) -below for how to configure this. - -### Borg hosting providers - -Most [Borg hosting -providers](https://torsion.org/borgmatic/#hosting-providers) include -monitoring and alerting as part of their offering. This gives you a dashboard -to check on all of your backups, and can alert you if the service doesn't hear -from borgmatic for a configured interval. - -### Consistency checks - -While not strictly part of monitoring, if you want confidence that your -backups are not only running but are restorable as well, you can configure -particular [consistency -checks](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#consistency-check-configuration) -or even script full [extract -tests](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/). - - -## Error hooks - -When an error occurs during a `create`, `prune`, `compact`, or `check` action, -borgmatic can run configurable shell commands to fire off custom error -notifications or take other actions, so you can get alerted as soon as -something goes wrong. Here's a not-so-useful example: - -```yaml -on_error: - - echo "Error while creating a backup or running a backup hook." -``` - -Prior to version 1.8.0 Put -this option in the `hooks:` section of your configuration. - -The `on_error` hook supports interpolating particular runtime variables into -the hook command. Here's an example that assumes you provide a separate shell -script to handle the alerting: - -```yaml -on_error: - - send-text-message.sh {configuration_filename} {repository} -``` - -In this example, when the error occurs, borgmatic interpolates runtime values -into the hook command: the borgmatic configuration filename and the path of -the repository. Here's the full set of supported variables you can use here: - - * `configuration_filename`: borgmatic configuration filename in which the - error occurred - * `repository`: path of the repository in which the error occurred (may be - blank if the error occurs in a hook) - * `error`: the error message itself - * `output`: output of the command that failed (may be blank if an error - occurred without running a command) - -Note that borgmatic runs the `on_error` hooks only for `create`, `prune`, -`compact`, or `check` actions/hooks in which an error occurs and not other -actions. borgmatic does not run `on_error` hooks if an error occurs within a -`before_everything` or `after_everything` hook. For more about hooks, see the -[borgmatic hooks -documentation](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/), -especially the security information. - -New in version 1.8.7 borgmatic -automatically escapes these interpolated values to prevent shell injection -attacks. One implication of this change is that you shouldn't wrap the -interpolated values in your own quotes, as that will interfere with the -quoting performed by borgmatic and result in your command receiving incorrect -arguments. For instance, this won't work: - - -```yaml -on_error: - # Don't do this! It won't work, as the {error} value is already quoted. - - send-text-message.sh "Uh oh: {error}" -``` - -Do this instead: - -```yaml -on_error: - - send-text-message.sh {error} -``` + * **Job runner alerts**: The easiest place to start is with failure alerts from + the [scheduled job + runner](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#autopilot) + (cron, systemd, etc.) that's running borgmatic. But note that if the job + doesn't even get scheduled (e.g. due to the job runner not running), you + probably won't get an alert at all! Still, this is a decent first line of + defense, especially when combined with some of the other approaches below. + * **Third-party monitoring services:** borgmatic integrates with these monitoring + services and libraries, pinging them as backups happen. The idea is that + you'll receive an alert when something goes wrong or when the service doesn't + hear from borgmatic for a configured interval (if supported). While these + services and libraries offer different features, you probably only need to + use one of them at most. See these documentation links for configuration + information: + * [Apprise](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#apprise-hook) + * [Cronhub](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook) + * [Cronitor](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook) + * [Grafana Loki](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#loki-hook) + * [Healthchecks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook) + * [ntfy](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#ntfy-hook) + * [PagerDuty](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook) + * [Pushover](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pushover-hook) + * [Sentry](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#sentry-hook) + * [Uptime Kuma](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#uptime-kuma-hook) + * [Zabbix](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#zabbix-hook) + * **Third-party monitoring software:** You can use traditional monitoring + software to consume borgmatic JSON output and track when the last successful + backup occurred. See [scripting + borgmatic](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#scripting-borgmatic) + below for how to configure this. + * **Borg hosting providers:** Some [Borg hosting + providers](https://torsion.org/borgmatic/#hosting-providers) include + monitoring and alerting as part of their offering. This gives you a dashboard + to check on all of your backups, and can alert you if the service doesn't + hear from borgmatic for a configured interval. + * **Consistency checks:** While not strictly part of monitoring, if you want + confidence that your backups are not only running but are restorable as well, + you can configure particular [consistency + checks](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#consistency-check-configuration) + or even script full [extract + tests](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/). + * **Commands run on error:** borgmatic's command hooks support running + arbitrary commands or scripts when borgmatic itself encounters an error + running your backups. So for instance, you can run a script to send yourself + a text message alert. But note that if borgmatic doesn't actually run, this + alert won't fire. See the [documentation on command hooks](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/) + for details. ## Healthchecks hook @@ -292,6 +207,27 @@ If you have any issues with the integration, [please contact us](https://torsion.org/borgmatic/#support-and-contributing). +### Sending logs + +New in version 1.9.14 borgmatic +logs are included in the payload data sent to PagerDuty. This means that +(truncated) borgmatic logs, including error messages, show up in the PagerDuty +incident UI and corresponding notification emails. + +You can customize the verbosity of the logs that are sent with borgmatic's +`--monitoring-verbosity` flag. The `--list` and `--stats` flags may also be of +use. See `borgmatic create --help` for more information. + +If you don't want any logs sent, you can disable this feature by setting +`send_logs` to `false`: + +```yaml +pagerduty: + integration_key: a177cad45bd374409f78906a810a3074 + send_logs: false +``` + + ## Pushover hook New in version 1.9.2 @@ -710,7 +646,10 @@ zabbix: - fail ``` -This hook requires the Zabbix server be running version 7.0+ +This hook requires the Zabbix server be running version 7.0. + +New in version 1.9.3 Zabbix 7.2+ +is supported as well. ### Authentication methods @@ -721,11 +660,19 @@ Authentication can be accomplished via `api_key` or both `username` and ### Items -The item to be updated can be chosen by either declaring the `itemid` or both -`host` and `key`. If all three are declared, only `itemid` is used. +borgmatic writes its monitoring updates to a particular Zabbix item, which +you'll need to create in advance. In the Zabbix web UI, [make a new item with a +Type of "Zabbix +trapper"](https://www.zabbix.com/documentation/current/en/manual/config/items/itemtypes/trapper) +and a named Key. The "Type of information" for the item should be "Text", and +"History" designates how much data you want to retain. -Keep in mind that `host` is referring to the "Host name" on the Zabbix server -and not the "Visual name". +When configuring borgmatic with this item to be updated, you can either declare +the `itemid` or both `host` and `key`. If all three are declared, only `itemid` +is used. + +Keep in mind that `host` refers to the "Host name" on the Zabbix server and not +the "Visual name". ## Scripting borgmatic diff --git a/docs/how-to/provide-your-passwords.md b/docs/how-to/provide-your-passwords.md index 60aa33b7..7ce3c8c3 100644 --- a/docs/how-to/provide-your-passwords.md +++ b/docs/how-to/provide-your-passwords.md @@ -19,6 +19,7 @@ encryption_passphrase: yourpassphrase But if you'd rather store them outside of borgmatic, whether for convenience or security reasons, read on. + ### Delegating to another application borgmatic supports calling another application such as a password manager to @@ -31,71 +32,262 @@ to provide the passphrase: encryption_passcommand: pass path/to/borg-passphrase ``` -Another example for [KeePassXC](https://keepassxc.org/): - -```yaml -encryption_passcommand: keepassxc-cli show --show-protected --attributes Password credentials.kdbx borg_passphrase -``` - -... where `borg_passphrase` is the title of the KeePassXC entry containing your -Borg encryption passphrase in its `Password` field. - New in version 1.9.9 Instead of letting Borg run the passcommand—potentially multiple times since borgmatic runs Borg multiple times—borgmatic now runs the passcommand itself and passes the -resulting passprhase securely to Borg via an anonymous pipe. This means you +resulting passphrase securely to Borg via an anonymous pipe. This means you should only ever get prompted for your password manager's passphrase at most once per borgmatic run. -### Using systemd service credentials +### systemd service credentials -Borgmatic supports using encrypted [credentials](https://systemd.io/CREDENTIALS/). +borgmatic supports reading encrypted [systemd +credentials](https://systemd.io/CREDENTIALS/). To use this feature, start by +saving your password as an encrypted credential to +`/etc/credstore.encrypted/borgmatic.pw`, e.g., -Save your password as an encrypted credential to `/etc/credstore.encrypted/borgmatic.pw`, e.g., - -``` -# systemd-ask-password -n | systemd-creds encrypt - /etc/credstore.encrypted/borgmatic.pw +```bash +systemd-ask-password -n | systemd-creds encrypt - /etc/credstore.encrypted/borgmatic.pw ``` -Then uncomment or use the following in your configuration file: +Then use the following in your configuration file: ```yaml -encryption_passcommand: "cat ${CREDENTIALS_DIRECTORY}/borgmatic.pw" +encryption_passphrase: "{credential systemd borgmatic.pw}" +``` + +Prior to version 1.9.10 You can +accomplish the same thing with this configuration: + +```yaml +encryption_passcommand: cat ${CREDENTIALS_DIRECTORY}/borgmatic.pw ``` Note that the name `borgmatic.pw` is hardcoded in the systemd service file. -To use multiple different passwords, save them as encrypted credentials to `/etc/credstore.encrypted/borgmatic/`, e.g., +The `{credential ...}` syntax works for several different options in a borgmatic +configuration file besides just `encryption_passphrase`. For instance, the +username, password, and API token options within database and monitoring hooks +support `{credential ...}`: +```yaml +postgresql_databases: + - name: invoices + username: postgres + password: "{credential systemd borgmatic_db1}" ``` -# mkdir /etc/credstore.encrypted/borgmatic -# systemd-ask-password -n | systemd-creds encrypt --name=borgmatic_backupserver1 - /etc/credstore.encrypted/borgmatic/backupserver1 -# systemd-ask-password -n | systemd-creds encrypt --name=borgmatic_pw2 - /etc/credstore.encrypted/borgmatic/pw2 + +For specifics about which options are supported, see the +[configuration +reference](https://torsion.org/borgmatic/docs/reference/configuration/). + +To use these credentials, you'll need to modify the borgmatic systemd service +file to support loading multiple credentials (assuming you need to load more +than one or anything not named `borgmatic.pw`). + +Start by saving each encrypted credentials to +`/etc/credstore.encrypted/borgmatic/`. E.g., + +```bash +mkdir /etc/credstore.encrypted/borgmatic +systemd-ask-password -n | systemd-creds encrypt --name=borgmatic_backupserver1 - /etc/credstore.encrypted/borgmatic/backupserver1 +systemd-ask-password -n | systemd-creds encrypt --name=borgmatic_pw2 - /etc/credstore.encrypted/borgmatic/pw2 ... ``` -Ensure that the file names, (e.g. `backupserver1`) match the corresponding part of -the `--name` option *after* the underscore (_), and that the part *before* +Ensure that the file names, (e.g. `backupserver1`) match the corresponding part +of the `--name` option *after* the underscore (_), and that the part *before* the underscore matches the directory name (e.g. `borgmatic`). Then, uncomment the appropriate line in the systemd service file: ``` -# systemctl edit borgmatic.service +systemctl edit borgmatic.service ... # Load multiple encrypted credentials. LoadCredentialEncrypted=borgmatic:/etc/credstore.encrypted/borgmatic/ ``` -Finally, use the following in your configuration file: +Finally, use something like the following in your borgmatic configuration file +for each option value you'd like to load from systemd: -``` -encryption_passcommand: "cat ${CREDENTIALS_DIRECTORY}/borgmatic_backupserver1" +```yaml +encryption_passphrase: "{credential systemd borgmatic_backupserver1}" ``` -Adjust `borgmatic_backupserver1` according to the name given to the credential -and the directory set in the service file. +Prior to version 1.9.10 Use the +following instead, but only for the `encryption_passcommand` option and +not other options: + +```yaml +encryption_passcommand: cat ${CREDENTIALS_DIRECTORY}/borgmatic_backupserver1 +``` + +Adjust `borgmatic_backupserver1` according to the name of the credential and the +directory set in the service file. + +Be aware that when using this systemd `{credential ...}` feature, you may no +longer be able to run certain borgmatic actions outside of the systemd service, +as the credentials are only available from within the context of that service. +So for instance, `borgmatic list` necessarily relies on the +`encryption_passphrase` in order to access the Borg repository, but `list` +shouldn't need to load any credentials for your database or monitoring hooks. + +The one exception is `borgmatic config validate`, which doesn't actually load +any credentials and should continue working anywhere. + + +### Container secrets + +New in version 1.9.11 When +running inside a container, borgmatic can read [Docker +secrets](https://docs.docker.com/compose/how-tos/use-secrets/) and [Podman +secrets](https://www.redhat.com/en/blog/new-podman-secrets-command). Creating +those secrets and passing them into your borgmatic container is outside the +scope of this documentation, but here's a simple example of that with [Docker +Compose](https://docs.docker.com/compose/): + +```yaml +services: + borgmatic: + # Use the actual image name of your borgmatic container here. + image: borgmatic:latest + secrets: + - borgmatic_passphrase +secrets: + borgmatic_passphrase: + file: /etc/borgmatic/passphrase.txt +``` + +This assumes there's a file on the host at `/etc/borgmatic/passphrase.txt` +containing your passphrase. Docker or Podman mounts the contents of that file +into a secret named `borgmatic_passphrase` in the borgmatic container at +`/run/secrets/`. + +Once your container secret is in place, you can consume it within your borgmatic +configuration file: + +```yaml +encryption_passphrase: "{credential container borgmatic_passphrase}" +``` + +This reads the secret securely from a file mounted at +`/run/secrets/borgmatic_passphrase` within the borgmatic container. + +The `{credential ...}` syntax works for several different options in a borgmatic +configuration file besides just `encryption_passphrase`. For instance, the +username, password, and API token options within database and monitoring hooks +support `{credential ...}`: + +```yaml +postgresql_databases: + - name: invoices + username: postgres + password: "{credential container borgmatic_db1}" +``` + +For specifics about which options are supported, see the +[configuration +reference](https://torsion.org/borgmatic/docs/reference/configuration/). + +You can also optionally override the `/run/secrets` directory that borgmatic reads secrets from +inside a container: + +```yaml +container: + secrets_directory: /path/to/secrets +``` + +But you should only need to do this for development or testing purposes. + + +### KeePassXC passwords + +New in version 1.9.11 borgmatic +supports reading passwords from the [KeePassXC](https://keepassxc.org/) password +manager. To use this feature, start by creating an entry in your KeePassXC +database, putting your password into the "Password" field of that entry and +making sure it's saved. + +Then, you can consume that password in your borgmatic configuration file. For +instance, if the entry's title is "borgmatic" and your KeePassXC database is +located at `/etc/keys.kdbx`, do this: + +```yaml +encryption_passphrase: "{credential keepassxc /etc/keys.kdbx borgmatic}" +``` + +But if the entry's title is multiple words like `borg pw`, you'll +need to quote it: + +```yaml +encryption_passphrase: "{credential keepassxc /etc/keys.kdbx 'borg pw'}" +``` + +With this in place, borgmatic runs the `keepassxc-cli` command to retrieve the +passphrase on demand. But note that `keepassxc-cli` will prompt for its own +passphrase in order to unlock its database, so be prepared to enter it when +running borgmatic. + +The `{credential ...}` syntax works for several different options in a borgmatic +configuration file besides just `encryption_passphrase`. For instance, the +username, password, and API token options within database and monitoring hooks +support `{credential ...}`: + +```yaml +postgresql_databases: + - name: invoices + username: postgres + password: "{credential keepassxc /etc/keys.kdbx database}" +``` + +For specifics about which options are supported, see the +[configuration +reference](https://torsion.org/borgmatic/docs/reference/configuration/). + +You can also optionally override the `keepassxc-cli` command that borgmatic calls to load +passwords: + +```yaml +keepassxc: + keepassxc_cli_command: /usr/local/bin/keepassxc-cli +``` + + +### File-based credentials + +New in version 1.9.11 borgmatic +supports reading credentials from arbitrary file paths. To use this feature, +start by writing your credential into a file that borgmatic has permission to +read. Take care not to include anything in the file other than your credential. +(borgmatic is smart enough to strip off a trailing newline though.) + +You can consume that credential file in your borgmatic configuration. For +instance, if your credential file is at `/credentials/borgmatic.txt`, do this: + +```yaml +encryption_passphrase: "{credential file /credentials/borgmatic.txt}" +``` + +With this in place, borgmatic reads the credential from the file path. + +The `{credential ...}` syntax works for several different options in a borgmatic +configuration file besides just `encryption_passphrase`. For instance, the +username, password, and API token options within database and monitoring hooks +support `{credential ...}`: + +```yaml +postgresql_databases: + - name: invoices + username: postgres + password: "{credential file /credentials/database.txt}" +``` + +For specifics about which options are supported, see the +[configuration +reference](https://torsion.org/borgmatic/docs/reference/configuration/). + ### Environment variable interpolation @@ -103,7 +295,15 @@ and the directory set in the service file. supports interpolating arbitrary environment variables directly into option values in your configuration file. That means you can instruct borgmatic to pull your repository passphrase, your database passwords, or any other option -values from environment variables. For instance: +values from environment variables. + +Be aware though that environment variables may be less secure than some of the +other approaches above for getting credentials into borgmatic. That's because +environment variables may be visible from within child processes and/or OS-level +process metadata. + +Here's an example of using an environment variable from borgmatic's +configuration file: ```yaml encryption_passphrase: ${YOUR_PASSPHRASE} @@ -165,6 +365,7 @@ can escape it with a backslash. For instance, if your password is literally encryption_passphrase: \${A}@! ``` + ## Related features Another way to override particular options within a borgmatic configuration @@ -177,9 +378,3 @@ Additionally, borgmatic action hooks support their own [variable interpolation](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/#variable-interpolation), although in that case it's for particular borgmatic runtime values rather than (only) environment variables. - -Lastly, if you do want to specify your passhprase directly within borgmatic -configuration, but you'd like to keep it in a separate file from your main -configuration, you can [use a configuration include or a merge -include](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-includes) -to pull in an external password. diff --git a/docs/how-to/set-up-backups.md b/docs/how-to/set-up-backups.md index 3965e12e..5871bd6b 100644 --- a/docs/how-to/set-up-backups.md +++ b/docs/how-to/set-up-backups.md @@ -296,6 +296,20 @@ skip_actions: - compact ``` +### Disabling default actions + +By default, running `borgmatic` without any arguments will perform the default +backup actions (create, prune, compact and check). If you want to disable this +behavior and require explicit actions to be specified, add the following to +your configuration: + +```yaml +default_actions: false +``` + +With this setting, running `borgmatic` without arguments will show the help +message instead of performing any actions. + ## Autopilot @@ -311,7 +325,6 @@ Then, from the directory where you downloaded it: ```bash sudo mv borgmatic /etc/cron.d/borgmatic -sudo chmod +x /etc/cron.d/borgmatic ``` If borgmatic is installed at a different location than diff --git a/docs/how-to/snapshot-your-filesystems.md b/docs/how-to/snapshot-your-filesystems.md index a9f2d267..018e6553 100644 --- a/docs/how-to/snapshot-your-filesystems.md +++ b/docs/how-to/snapshot-your-filesystems.md @@ -54,8 +54,8 @@ You have a couple of options for borgmatic to find and backup your ZFS datasets: * For any dataset you'd like backed up, add its mount point to borgmatic's `source_directories` option. * New in version 1.9.6 Or - include the mount point with borgmatic's `patterns` or `patterns_from` - options. + include the mount point as a root pattern with borgmatic's `patterns` or + `patterns_from` options. * Or set the borgmatic-specific user property `org.torsion.borgmatic:backup=auto` onto your dataset, e.g. by running `zfs set org.torsion.borgmatic:backup=auto datasetname`. Then borgmatic can find @@ -65,6 +65,11 @@ If you have multiple borgmatic configuration files with ZFS enabled, and you'd like particular datasets to be backed up only for particular configuration files, use the `source_directories` option instead of the user property. +New in version 1.9.11 borgmatic +won't snapshot datasets with the `canmount=off` property, which is often set on +datasets that only serve as a container for other datasets. Use `zfs get +canmount datasetname` to see the `canmount` value for a dataset. + During a backup, borgmatic automatically snapshots these discovered datasets (non-recursively), temporarily mounts the snapshots within its [runtime directory](https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#runtime-directory), @@ -143,38 +148,40 @@ feedback](https://torsion.org/borgmatic/#issues) you have on this feature. #### Subvolume discovery -For any subvolume you'd like backed up, add its path to borgmatic's -`source_directories` option. +For any read-write subvolume you'd like backed up, add its mount point path to +borgmatic's `source_directories` option. Btrfs does not support snapshotting +read-only subvolumes. New in version 1.9.6 Or include -the mount point with borgmatic's `patterns` or `patterns_from` options. +the mount point as a root pattern with borgmatic's `patterns` or `patterns_from` +options. During a backup, borgmatic snapshots these subvolumes (non-recursively) and includes the snapshotted files in the paths sent to Borg. borgmatic is also responsible for cleaning up (deleting) these snapshots after a backup completes. borgmatic is smart enough to look at the parent (and grandparent, etc.) -directories of each of your `source_directories` to discover any subvolumes. -For instance, let's say you add `/var/log` and `/var/lib` to your source -directories, but `/var` is a subvolume. borgmatic will discover that and -snapshot `/var` accordingly. This also works even with nested subvolumes; +directories of each of your `source_directories` to discover any subvolumes. For +instance, let's say you add `/var/log` and `/var/lib` to your source +directories, but `/var` is a subvolume mount point. borgmatic will discover that +and snapshot `/var` accordingly. This also works even with nested subvolumes; borgmatic selects the subvolume that's the "closest" parent to your source directories. New in version 1.9.6 When using [patterns](https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-help-patterns), the initial portion of a pattern's path that you intend borgmatic to match -against a subvolume can't have globs or other non-literal characters in it—or it -won't actually match. For instance, a subvolume of `/var` would match a pattern -of `+ fm:/var/*/data`, but borgmatic isn't currently smart enough to match -`/var` to a pattern like `+ fm:/v*/lib/data`. +against a subvolume mount point can't have globs or other non-literal characters +in it—or it won't actually match. For instance, a subvolume mount point of +`/var` would match a pattern of `+ fm:/var/*/data`, but borgmatic isn't +currently smart enough to match `/var` to a pattern like `+ fm:/v*/lib/data`. -Additionally, borgmatic rewrites the snapshot file paths so that they appear -at their original subvolume locations in a Borg archive. For instance, if your -subvolume exists at `/var/subvolume`, then the snapshotted files will appear +Additionally, borgmatic rewrites the snapshot file paths so that they appear at +their original subvolume locations in a Borg archive. For instance, if your +subvolume is mounted at `/var/subvolume`, then the snapshotted files will appear in an archive at `/var/subvolume` as well—even if borgmatic has to mount the -snapshot somewhere in `/var/subvolume/.borgmatic-snapshot-1234/` to perform -the backup. +snapshot somewhere in `/var/subvolume/.borgmatic-snapshot-1234/` to perform the +backup. With Borg version 1.2 and earlierSnapshotted files are instead stored at a path dependent on the @@ -199,6 +206,14 @@ Volume Manager) and sending those snapshots to Borg for backup. LVM isn't itself a filesystem, but it can take snapshots at the layer right below your filesystem. +Note that, due to Borg being a file-level backup, this feature is really only +suitable for filesystems, not whole disk or raw images containing multiple +filesystems (for example, if you're using a LVM volume to run a Windows +KVM that contains an MBR, partitions, etc.). + +In those cases, you can omit the `lvm:` option and use Borg's own support for +[image backup](https://borgbackup.readthedocs.io/en/stable/deployment/image-backup.html). + To use this feature, first you need one or more mounted LVM logical volumes. Then, enable LVM within borgmatic by adding the following line to your configuration file: @@ -252,7 +267,8 @@ For any logical volume you'd like backed up, add its mount point to borgmatic's `source_directories` option. New in version 1.9.6 Or include -the mount point with borgmatic's `patterns` or `patterns_from` options. +the mount point as a root pattern with borgmatic's `patterns` or `patterns_from` +options. During a backup, borgmatic automatically snapshots these discovered logical volumes (non-recursively), temporarily mounts the snapshots within its [runtime diff --git a/docs/static/docker.png b/docs/static/docker.png new file mode 100644 index 00000000..a3a192ab Binary files /dev/null and b/docs/static/docker.png differ diff --git a/docs/static/keepassxc.png b/docs/static/keepassxc.png new file mode 100644 index 00000000..059e7af9 Binary files /dev/null and b/docs/static/keepassxc.png differ diff --git a/docs/static/podman.png b/docs/static/podman.png new file mode 100644 index 00000000..9d5d7a41 Binary files /dev/null and b/docs/static/podman.png differ diff --git a/docs/static/pushover.png b/docs/static/pushover.png index e0ef72a9..4d49fc73 100644 Binary files a/docs/static/pushover.png and b/docs/static/pushover.png differ diff --git a/docs/static/systemd.png b/docs/static/systemd.png new file mode 100644 index 00000000..ced38c9e Binary files /dev/null and b/docs/static/systemd.png differ diff --git a/pyproject.toml b/pyproject.toml index 92102e3f..d01b6f8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "borgmatic" -version = "1.9.10.dev0" +version = "2.0.0dev0" authors = [ { name="Dan Helfman", email="witten@torsion.org" }, ] diff --git a/tests/end-to-end/commands/fake_btrfs.py b/tests/end-to-end/commands/fake_btrfs.py index ee969597..b23fc553 100644 --- a/tests/end-to-end/commands/fake_btrfs.py +++ b/tests/end-to-end/commands/fake_btrfs.py @@ -24,7 +24,14 @@ def parse_arguments(*unparsed_arguments): delete_parser = subvolume_subparser.add_parser('delete') delete_parser.add_argument('snapshot_path') - return global_parser.parse_args(unparsed_arguments) + property_parser = action_parsers.add_parser('property') + property_subparser = property_parser.add_subparsers(dest='subaction') + get_parser = property_subparser.add_parser('get') + get_parser.add_argument('-t', dest='type') + get_parser.add_argument('subvolume_path') + get_parser.add_argument('property_name') + + return (global_parser, global_parser.parse_args(unparsed_arguments)) BUILTIN_SUBVOLUME_LIST_LINES = ( @@ -60,9 +67,13 @@ def print_subvolume_list(arguments, snapshot_paths): def main(): - arguments = parse_arguments(*sys.argv[1:]) + (global_parser, arguments) = parse_arguments(*sys.argv[1:]) snapshot_paths = load_snapshots() + if not hasattr(arguments, 'subaction'): + global_parser.print_help() + sys.exit(1) + if arguments.subaction == 'list': print_subvolume_list(arguments, snapshot_paths) elif arguments.subaction == 'snapshot': @@ -84,6 +95,8 @@ def main(): if snapshot_path.endswith('/' + arguments.snapshot_path) ] save_snapshots(snapshot_paths) + elif arguments.action == 'property' and arguments.subaction == 'get': + print(f'{arguments.property_name}=false') if __name__ == '__main__': diff --git a/tests/end-to-end/commands/fake_keepassxc_cli.py b/tests/end-to-end/commands/fake_keepassxc_cli.py new file mode 100644 index 00000000..4c22c490 --- /dev/null +++ b/tests/end-to-end/commands/fake_keepassxc_cli.py @@ -0,0 +1,29 @@ +import argparse +import sys + + +def parse_arguments(*unparsed_arguments): + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument('command') + parser.add_argument('--show-protected', action='store_true') + parser.add_argument('--attributes') + parser.add_argument('database_path') + parser.add_argument('attribute_name') + + return parser.parse_args(unparsed_arguments) + + +def main(): + arguments = parse_arguments(*sys.argv[1:]) + + assert arguments.command == 'show' + assert arguments.show_protected + assert arguments.attributes == 'Password' + assert arguments.database_path.endswith('.kdbx') + assert arguments.attribute_name + + print('test') + + +if __name__ == '__main__': + main() diff --git a/tests/end-to-end/commands/fake_zfs.py b/tests/end-to-end/commands/fake_zfs.py index 61f8ae5e..3c13f4d5 100644 --- a/tests/end-to-end/commands/fake_zfs.py +++ b/tests/end-to-end/commands/fake_zfs.py @@ -27,6 +27,7 @@ BUILTIN_DATASETS = ( 'used': '256K', 'avail': '23.7M', 'refer': '25K', + 'canmount': 'on', 'mountpoint': '/pool', }, { @@ -34,6 +35,7 @@ BUILTIN_DATASETS = ( 'used': '256K', 'avail': '23.7M', 'refer': '25K', + 'canmount': 'on', 'mountpoint': '/pool/dataset', }, ) diff --git a/tests/end-to-end/hooks/credential/test_container.py b/tests/end-to-end/hooks/credential/test_container.py new file mode 100644 index 00000000..6279f522 --- /dev/null +++ b/tests/end-to-end/hooks/credential/test_container.py @@ -0,0 +1,68 @@ +import json +import os +import shutil +import subprocess +import sys +import tempfile + + +def generate_configuration(config_path, repository_path, secrets_directory): + ''' + Generate borgmatic configuration into a file at the config path, and update the defaults so as + to work for testing, including updating the source directories, injecting the given repository + path, and tacking on an encryption passphrase loaded from container secrets in the given secrets + directory. + ''' + subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' ')) + config = ( + open(config_path) + .read() + .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path) + .replace('- path: /mnt/backup', '') + .replace('label: local', '') + .replace('- /home/user/path with spaces', '') + .replace('- /home', f'- {config_path}') + .replace('- /etc', '') + .replace('- /var/log/syslog*', '') + + '\nencryption_passphrase: "{credential container mysecret}"' + + f'\ncontainer:\n secrets_directory: {secrets_directory}' + ) + config_file = open(config_path, 'w') + config_file.write(config) + config_file.close() + + +def test_container_secret(): + # Create a Borg repository. + temporary_directory = tempfile.mkdtemp() + repository_path = os.path.join(temporary_directory, 'test.borg') + + original_working_directory = os.getcwd() + os.chdir(temporary_directory) + + try: + config_path = os.path.join(temporary_directory, 'test.yaml') + generate_configuration(config_path, repository_path, secrets_directory=temporary_directory) + + secret_path = os.path.join(temporary_directory, 'mysecret') + with open(secret_path, 'w') as secret_file: + secret_file.write('test') + + subprocess.check_call( + f'borgmatic -v 2 --config {config_path} repo-create --encryption repokey'.split(' '), + ) + + # Run borgmatic to generate a backup archive, and then list it to make sure it exists. + subprocess.check_call( + f'borgmatic --config {config_path}'.split(' '), + ) + output = subprocess.check_output( + f'borgmatic --config {config_path} list --json'.split(' '), + ).decode(sys.stdout.encoding) + parsed_output = json.loads(output) + + assert len(parsed_output) == 1 + assert len(parsed_output[0]['archives']) == 1 + finally: + os.chdir(original_working_directory) + shutil.rmtree(temporary_directory) diff --git a/tests/end-to-end/hooks/credential/test_file.py b/tests/end-to-end/hooks/credential/test_file.py new file mode 100644 index 00000000..0ae9b37a --- /dev/null +++ b/tests/end-to-end/hooks/credential/test_file.py @@ -0,0 +1,68 @@ +import json +import os +import shutil +import subprocess +import sys +import tempfile + + +def generate_configuration(config_path, repository_path, credential_path): + ''' + Generate borgmatic configuration into a file at the config path, and update the defaults so as + to work for testing, including updating the source directories, injecting the given repository + path, and tacking on an encryption passphrase loaded from file at the given credential path. + ''' + subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' ')) + config = ( + open(config_path) + .read() + .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path) + .replace('- path: /mnt/backup', '') + .replace('label: local', '') + .replace('- /home/user/path with spaces', '') + .replace('- /home', f'- {config_path}') + .replace('- /etc', '') + .replace('- /var/log/syslog*', '') + + '\nencryption_passphrase: "{credential file ' + + credential_path + + '}"' + ) + config_file = open(config_path, 'w') + config_file.write(config) + config_file.close() + + +def test_file_credential(): + # Create a Borg repository. + temporary_directory = tempfile.mkdtemp() + repository_path = os.path.join(temporary_directory, 'test.borg') + + original_working_directory = os.getcwd() + os.chdir(temporary_directory) + + try: + config_path = os.path.join(temporary_directory, 'test.yaml') + credential_path = os.path.join(temporary_directory, 'mycredential') + generate_configuration(config_path, repository_path, credential_path) + + with open(credential_path, 'w') as credential_file: + credential_file.write('test') + + subprocess.check_call( + f'borgmatic -v 2 --config {config_path} repo-create --encryption repokey'.split(' '), + ) + + # Run borgmatic to generate a backup archive, and then list it to make sure it exists. + subprocess.check_call( + f'borgmatic --config {config_path}'.split(' '), + ) + output = subprocess.check_output( + f'borgmatic --config {config_path} list --json'.split(' '), + ).decode(sys.stdout.encoding) + parsed_output = json.loads(output) + + assert len(parsed_output) == 1 + assert len(parsed_output[0]['archives']) == 1 + finally: + os.chdir(original_working_directory) + shutil.rmtree(temporary_directory) diff --git a/tests/end-to-end/hooks/credential/test_keepassxc.py b/tests/end-to-end/hooks/credential/test_keepassxc.py new file mode 100644 index 00000000..e990458f --- /dev/null +++ b/tests/end-to-end/hooks/credential/test_keepassxc.py @@ -0,0 +1,67 @@ +import json +import os +import shutil +import subprocess +import sys +import tempfile + + +def generate_configuration(config_path, repository_path): + ''' + Generate borgmatic configuration into a file at the config path, and update the defaults so as + to work for testing, including updating the source directories, injecting the given repository + path, and tacking on an encryption passphrase loaded from keepassxc-cli. + ''' + subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' ')) + config = ( + open(config_path) + .read() + .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path) + .replace('- path: /mnt/backup', '') + .replace('label: local', '') + .replace('- /home/user/path with spaces', '') + .replace('- /home', f'- {config_path}') + .replace('- /etc', '') + .replace('- /var/log/syslog*', '') + + '\nencryption_passphrase: "{credential keepassxc keys.kdbx mypassword}"' + + '\nkeepassxc:\n keepassxc_cli_command: python3 /app/tests/end-to-end/commands/fake_keepassxc_cli.py' + ) + config_file = open(config_path, 'w') + config_file.write(config) + config_file.close() + + +def test_keepassxc_password(): + # Create a Borg repository. + temporary_directory = tempfile.mkdtemp() + repository_path = os.path.join(temporary_directory, 'test.borg') + + original_working_directory = os.getcwd() + os.chdir(temporary_directory) + + try: + config_path = os.path.join(temporary_directory, 'test.yaml') + generate_configuration(config_path, repository_path) + + database_path = os.path.join(temporary_directory, 'keys.kdbx') + with open(database_path, 'w') as database_file: + database_file.write('fake KeePassXC database to pacify file existence check') + + subprocess.check_call( + f'borgmatic -v 2 --config {config_path} repo-create --encryption repokey'.split(' '), + ) + + # Run borgmatic to generate a backup archive, and then list it to make sure it exists. + subprocess.check_call( + f'borgmatic --config {config_path}'.split(' '), + ) + output = subprocess.check_output( + f'borgmatic --config {config_path} list --json'.split(' '), + ).decode(sys.stdout.encoding) + parsed_output = json.loads(output) + + assert len(parsed_output) == 1 + assert len(parsed_output[0]['archives']) == 1 + finally: + os.chdir(original_working_directory) + shutil.rmtree(temporary_directory) diff --git a/tests/end-to-end/hooks/credential/test_systemd.py b/tests/end-to-end/hooks/credential/test_systemd.py new file mode 100644 index 00000000..9a263398 --- /dev/null +++ b/tests/end-to-end/hooks/credential/test_systemd.py @@ -0,0 +1,69 @@ +import json +import os +import shutil +import subprocess +import sys +import tempfile + + +def generate_configuration(config_path, repository_path): + ''' + Generate borgmatic configuration into a file at the config path, and update the defaults so as + to work for testing, including updating the source directories, injecting the given repository + path, and tacking on an encryption passphrase loaded from systemd. + ''' + subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' ')) + config = ( + open(config_path) + .read() + .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path) + .replace('- path: /mnt/backup', '') + .replace('label: local', '') + .replace('- /home/user/path with spaces', '') + .replace('- /home', f'- {config_path}') + .replace('- /etc', '') + .replace('- /var/log/syslog*', '') + + '\nencryption_passphrase: "{credential systemd mycredential}"' + ) + config_file = open(config_path, 'w') + config_file.write(config) + config_file.close() + + +def test_systemd_credential(): + # Create a Borg repository. + temporary_directory = tempfile.mkdtemp() + repository_path = os.path.join(temporary_directory, 'test.borg') + + original_working_directory = os.getcwd() + os.chdir(temporary_directory) + + try: + config_path = os.path.join(temporary_directory, 'test.yaml') + generate_configuration(config_path, repository_path) + + credential_path = os.path.join(temporary_directory, 'mycredential') + with open(credential_path, 'w') as credential_file: + credential_file.write('test') + + subprocess.check_call( + f'borgmatic -v 2 --config {config_path} repo-create --encryption repokey'.split(' '), + env=dict(os.environ, **{'CREDENTIALS_DIRECTORY': temporary_directory}), + ) + + # Run borgmatic to generate a backup archive, and then list it to make sure it exists. + subprocess.check_call( + f'borgmatic --config {config_path}'.split(' '), + env=dict(os.environ, **{'CREDENTIALS_DIRECTORY': temporary_directory}), + ) + output = subprocess.check_output( + f'borgmatic --config {config_path} list --json'.split(' '), + env=dict(os.environ, **{'CREDENTIALS_DIRECTORY': temporary_directory}), + ).decode(sys.stdout.encoding) + parsed_output = json.loads(output) + + assert len(parsed_output) == 1 + assert len(parsed_output[0]['archives']) == 1 + finally: + os.chdir(original_working_directory) + shutil.rmtree(temporary_directory) diff --git a/tests/end-to-end/test_config_flag.py b/tests/end-to-end/test_config_flag.py new file mode 100644 index 00000000..279a1c6e --- /dev/null +++ b/tests/end-to-end/test_config_flag.py @@ -0,0 +1,55 @@ +import os +import shlex +import shutil +import subprocess +import tempfile + + +def generate_configuration(config_path): + ''' + Generate borgmatic configuration into a file at the config path, and update the defaults so as + to work for testing (including injecting the given repository path and tacking on an encryption + passphrase). But don't actually set the repository path, as that's done on the command-line + below. + ''' + subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' ')) + config = ( + open(config_path) + .read() + .replace('- ssh://user@backupserver/./{fqdn}', '') # noqa: FS003 + .replace('- /var/local/backups/local.borg', '') + .replace('- /home/user/path with spaces', '') + .replace('- /home', f'- {config_path}') + .replace('- /etc', '') + .replace('- /var/log/syslog*', '') + + 'encryption_passphrase: "test"' + ) + config_file = open(config_path, 'w') + config_file.write(config) + config_file.close() + + +def test_config_flags_do_not_error(): + temporary_directory = tempfile.mkdtemp() + repository_path = os.path.join(temporary_directory, 'test.borg') + + original_working_directory = os.getcwd() + + try: + config_path = os.path.join(temporary_directory, 'test.yaml') + generate_configuration(config_path) + + subprocess.check_call( + shlex.split( + f'borgmatic -v 2 --config {config_path} --repositories "[{{path: {repository_path}, label: repo}}]" repo-create --encryption repokey' + ) + ) + + subprocess.check_call( + shlex.split( + f'borgmatic create --config {config_path} --repositories[0].path "{repository_path}"' + ) + ) + finally: + os.chdir(original_working_directory) + shutil.rmtree(temporary_directory) diff --git a/tests/integration/borg/test_commands.py b/tests/integration/borg/test_commands.py index 4e9261e8..80de7dcd 100644 --- a/tests/integration/borg/test_commands.py +++ b/tests/integration/borg/test_commands.py @@ -53,7 +53,7 @@ def fuzz_argument(arguments, argument_name): def test_transfer_archives_command_does_not_duplicate_flags_or_raise(): arguments = borgmatic.commands.arguments.parse_arguments( - 'transfer', '--source-repository', 'foo' + {}, 'transfer', '--source-repository', 'foo' )['transfer'] flexmock(borgmatic.borg.transfer).should_receive('execute_command').replace_with( assert_command_does_not_duplicate_flags @@ -74,7 +74,7 @@ def test_transfer_archives_command_does_not_duplicate_flags_or_raise(): def test_prune_archives_command_does_not_duplicate_flags_or_raise(): - arguments = borgmatic.commands.arguments.parse_arguments('prune')['prune'] + arguments = borgmatic.commands.arguments.parse_arguments({}, 'prune')['prune'] flexmock(borgmatic.borg.prune).should_receive('execute_command').replace_with( assert_command_does_not_duplicate_flags ) @@ -94,7 +94,7 @@ def test_prune_archives_command_does_not_duplicate_flags_or_raise(): def test_mount_archive_command_does_not_duplicate_flags_or_raise(): - arguments = borgmatic.commands.arguments.parse_arguments('mount', '--mount-point', 'tmp')[ + arguments = borgmatic.commands.arguments.parse_arguments({}, 'mount', '--mount-point', 'tmp')[ 'mount' ] flexmock(borgmatic.borg.mount).should_receive('execute_command').replace_with( @@ -116,7 +116,7 @@ def test_mount_archive_command_does_not_duplicate_flags_or_raise(): def test_make_list_command_does_not_duplicate_flags_or_raise(): - arguments = borgmatic.commands.arguments.parse_arguments('list')['list'] + arguments = borgmatic.commands.arguments.parse_arguments({}, 'list')['list'] for argument_name in dir(arguments): if argument_name.startswith('_'): @@ -134,7 +134,7 @@ def test_make_list_command_does_not_duplicate_flags_or_raise(): def test_make_repo_list_command_does_not_duplicate_flags_or_raise(): - arguments = borgmatic.commands.arguments.parse_arguments('repo-list')['repo-list'] + arguments = borgmatic.commands.arguments.parse_arguments({}, 'repo-list')['repo-list'] for argument_name in dir(arguments): if argument_name.startswith('_'): @@ -152,7 +152,7 @@ def test_make_repo_list_command_does_not_duplicate_flags_or_raise(): def test_display_archives_info_command_does_not_duplicate_flags_or_raise(): - arguments = borgmatic.commands.arguments.parse_arguments('info')['info'] + arguments = borgmatic.commands.arguments.parse_arguments({}, 'info')['info'] flexmock(borgmatic.borg.info).should_receive('execute_command_and_capture_output').replace_with( assert_command_does_not_duplicate_flags ) diff --git a/tests/integration/commands/completion/test_actions.py b/tests/integration/commands/completion/test_actions.py index 2e6fde9b..52dfd268 100644 --- a/tests/integration/commands/completion/test_actions.py +++ b/tests/integration/commands/completion/test_actions.py @@ -1,4 +1,5 @@ import borgmatic.commands.arguments +import borgmatic.config.validate from borgmatic.commands.completion import actions as module @@ -7,7 +8,10 @@ def test_available_actions_uses_only_subactions_for_action_with_subactions(): unused_global_parser, action_parsers, unused_combined_parser, - ) = borgmatic.commands.arguments.make_parsers() + ) = borgmatic.commands.arguments.make_parsers( + schema=borgmatic.config.validate.load_schema(borgmatic.config.validate.schema_filename()), + unparsed_arguments=(), + ) actions = module.available_actions(action_parsers, 'config') @@ -20,7 +24,10 @@ def test_available_actions_omits_subactions_for_action_without_subactions(): unused_global_parser, action_parsers, unused_combined_parser, - ) = borgmatic.commands.arguments.make_parsers() + ) = borgmatic.commands.arguments.make_parsers( + schema=borgmatic.config.validate.load_schema(borgmatic.config.validate.schema_filename()), + unparsed_arguments=(), + ) actions = module.available_actions(action_parsers, 'list') diff --git a/tests/integration/commands/test_arguments.py b/tests/integration/commands/test_arguments.py index 4399c867..b3ffef0d 100644 --- a/tests/integration/commands/test_arguments.py +++ b/tests/integration/commands/test_arguments.py @@ -4,11 +4,144 @@ from flexmock import flexmock from borgmatic.commands import arguments as module +def test_make_argument_description_with_object_adds_example(): + assert ( + module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'object', + 'example': {'bar': 'baz'}, + }, + flag_name='flag', + ) + # Apparently different versions of ruamel.yaml serialize this + # differently. + in ('Thing. Example value: "bar: baz"' 'Thing. Example value: "{bar: baz}"') + ) + + +def test_make_argument_description_with_array_adds_example(): + assert ( + module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'array', + 'example': [1, '- foo', {'bar': 'baz'}], + }, + flag_name='flag', + ) + # Apparently different versions of ruamel.yaml serialize this + # differently. + in ( + 'Thing. Example value: "[1, \'- foo\', bar: baz]"' + 'Thing. Example value: "[1, \'- foo\', {bar: baz}]"' + ) + ) + + +def test_add_array_element_arguments_adds_arguments_for_array_index_flags(): + parser = module.ArgumentParser(allow_abbrev=False, add_help=False) + arguments_group = parser.add_argument_group('arguments') + arguments_group.add_argument( + '--foo[0].val', + action='store_true', + dest='--foo[0].val', + ) + + flexmock(arguments_group).should_receive('add_argument').with_args( + '--foo[25].val', + action='store_true', + default=False, + dest='foo[25].val', + required=object, + ).once() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo[25].val', 'fooval', '--bar[1].val', 'barval'), + flag_name='foo[0].val', + ) + + +def test_add_arguments_from_schema_with_nested_object_adds_flag_for_each_option(): + parser = module.ArgumentParser(allow_abbrev=False, add_help=False) + arguments_group = parser.add_argument_group('arguments') + flexmock(arguments_group).should_receive('add_argument').with_args( + '--foo.bar', + type=int, + metavar='BAR', + help='help 1', + ).once() + flexmock(arguments_group).should_receive('add_argument').with_args( + '--foo.baz', + type=str, + metavar='BAZ', + help='help 2', + ).once() + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'object', + 'properties': { + 'bar': {'type': 'integer', 'description': 'help 1'}, + 'baz': {'type': 'string', 'description': 'help 2'}, + }, + } + }, + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_array_and_nested_object_adds_multiple_flags(): + parser = module.ArgumentParser(allow_abbrev=False, add_help=False) + arguments_group = parser.add_argument_group('arguments') + flexmock(arguments_group).should_receive('add_argument').with_args( + '--foo[0].bar', + type=int, + metavar='BAR', + help=object, + ).once() + flexmock(arguments_group).should_receive('add_argument').with_args( + '--foo', + type=str, + metavar='FOO', + help='help 2', + ).once() + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'bar': { + 'type': 'integer', + 'description': 'help 1', + } + }, + }, + 'description': 'help 2', + } + }, + }, + unparsed_arguments=(), + ) + + def test_parse_arguments_with_no_arguments_uses_defaults(): config_paths = ['default'] flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) - arguments = module.parse_arguments() + arguments = module.parse_arguments({}) global_arguments = arguments['global'] assert global_arguments.config_paths == config_paths @@ -21,7 +154,7 @@ def test_parse_arguments_with_no_arguments_uses_defaults(): def test_parse_arguments_with_multiple_config_flags_parses_as_list(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--config', 'myconfig', '--config', 'otherconfig') + arguments = module.parse_arguments({}, '--config', 'myconfig', '--config', 'otherconfig') global_arguments = arguments['global'] assert global_arguments.config_paths == ['myconfig', 'otherconfig'] @@ -34,7 +167,7 @@ def test_parse_arguments_with_multiple_config_flags_parses_as_list(): def test_parse_arguments_with_action_after_config_path_omits_action(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--config', 'myconfig', 'list', '--json') + arguments = module.parse_arguments({}, '--config', 'myconfig', 'list', '--json') global_arguments = arguments['global'] assert global_arguments.config_paths == ['myconfig'] @@ -45,7 +178,9 @@ def test_parse_arguments_with_action_after_config_path_omits_action(): def test_parse_arguments_with_action_after_config_path_omits_aliased_action(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey') + arguments = module.parse_arguments( + {}, '--config', 'myconfig', 'init', '--encryption', 'repokey' + ) global_arguments = arguments['global'] assert global_arguments.config_paths == ['myconfig'] @@ -56,7 +191,7 @@ def test_parse_arguments_with_action_after_config_path_omits_aliased_action(): def test_parse_arguments_with_action_and_positional_arguments_after_config_path_omits_action_and_arguments(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--config', 'myconfig', 'borg', 'key', 'export') + arguments = module.parse_arguments({}, '--config', 'myconfig', 'borg', 'key', 'export') global_arguments = arguments['global'] assert global_arguments.config_paths == ['myconfig'] @@ -68,7 +203,7 @@ def test_parse_arguments_with_verbosity_overrides_default(): config_paths = ['default'] flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) - arguments = module.parse_arguments('--verbosity', '1') + arguments = module.parse_arguments({}, '--verbosity', '1') global_arguments = arguments['global'] assert global_arguments.config_paths == config_paths @@ -82,7 +217,7 @@ def test_parse_arguments_with_syslog_verbosity_overrides_default(): config_paths = ['default'] flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) - arguments = module.parse_arguments('--syslog-verbosity', '2') + arguments = module.parse_arguments({}, '--syslog-verbosity', '2') global_arguments = arguments['global'] assert global_arguments.config_paths == config_paths @@ -96,7 +231,7 @@ def test_parse_arguments_with_log_file_verbosity_overrides_default(): config_paths = ['default'] flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths) - arguments = module.parse_arguments('--log-file-verbosity', '-1') + arguments = module.parse_arguments({}, '--log-file-verbosity', '-1') global_arguments = arguments['global'] assert global_arguments.config_paths == config_paths @@ -109,7 +244,7 @@ def test_parse_arguments_with_log_file_verbosity_overrides_default(): def test_parse_arguments_with_single_override_parses(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--override', 'foo.bar=baz') + arguments = module.parse_arguments({}, '--override', 'foo.bar=baz') global_arguments = arguments['global'] assert global_arguments.overrides == ['foo.bar=baz'] @@ -119,7 +254,7 @@ def test_parse_arguments_with_multiple_overrides_flags_parses(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) arguments = module.parse_arguments( - '--override', 'foo.bar=baz', '--override', 'foo.quux=7', '--override', 'this.that=8' + {}, '--override', 'foo.bar=baz', '--override', 'foo.quux=7', '--override', 'this.that=8' ) global_arguments = arguments['global'] @@ -127,7 +262,7 @@ def test_parse_arguments_with_multiple_overrides_flags_parses(): def test_parse_arguments_with_list_json_overrides_default(): - arguments = module.parse_arguments('list', '--json') + arguments = module.parse_arguments({}, 'list', '--json') assert 'list' in arguments assert arguments['list'].json is True @@ -136,7 +271,7 @@ def test_parse_arguments_with_list_json_overrides_default(): def test_parse_arguments_with_no_actions_defaults_to_all_actions_enabled(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments() + arguments = module.parse_arguments({}) assert 'prune' in arguments assert 'create' in arguments @@ -146,14 +281,14 @@ def test_parse_arguments_with_no_actions_defaults_to_all_actions_enabled(): def test_parse_arguments_with_no_actions_passes_argument_to_relevant_actions(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--stats', '--list') + arguments = module.parse_arguments({}, '--stats', '--list') assert 'prune' in arguments - assert arguments['prune'].stats - assert arguments['prune'].list_archives + assert arguments['prune'].statistics + assert arguments['prune'].list_details assert 'create' in arguments - assert arguments['create'].stats - assert arguments['create'].list_files + assert arguments['create'].statistics + assert arguments['create'].list_details assert 'check' in arguments @@ -161,7 +296,7 @@ def test_parse_arguments_with_help_and_no_actions_shows_global_help(capsys): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit) as exit: - module.parse_arguments('--help') + module.parse_arguments({}, '--help') assert exit.value.code == 0 captured = capsys.readouterr() @@ -173,7 +308,7 @@ def test_parse_arguments_with_help_and_action_shows_action_help(capsys): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit) as exit: - module.parse_arguments('create', '--help') + module.parse_arguments({}, 'create', '--help') assert exit.value.code == 0 captured = capsys.readouterr() @@ -185,7 +320,7 @@ def test_parse_arguments_with_help_and_action_shows_action_help(capsys): def test_parse_arguments_with_action_before_global_options_parses_options(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('prune', '--verbosity', '2') + arguments = module.parse_arguments({}, 'prune', '--verbosity', '2') assert 'prune' in arguments assert arguments['global'].verbosity == 2 @@ -194,7 +329,7 @@ def test_parse_arguments_with_action_before_global_options_parses_options(): def test_parse_arguments_with_global_options_before_action_parses_options(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('--verbosity', '2', 'prune') + arguments = module.parse_arguments({}, '--verbosity', '2', 'prune') assert 'prune' in arguments assert arguments['global'].verbosity == 2 @@ -203,7 +338,7 @@ def test_parse_arguments_with_global_options_before_action_parses_options(): def test_parse_arguments_with_prune_action_leaves_other_actions_disabled(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('prune') + arguments = module.parse_arguments({}, 'prune') assert 'prune' in arguments assert 'create' not in arguments @@ -213,7 +348,7 @@ def test_parse_arguments_with_prune_action_leaves_other_actions_disabled(): def test_parse_arguments_with_multiple_actions_leaves_other_action_disabled(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - arguments = module.parse_arguments('create', 'check') + arguments = module.parse_arguments({}, 'create', 'check') assert 'prune' not in arguments assert 'create' in arguments @@ -224,60 +359,53 @@ def test_parse_arguments_disallows_invalid_argument(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--posix-me-harder') + module.parse_arguments({}, '--posix-me-harder') def test_parse_arguments_disallows_encryption_mode_without_init(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--config', 'myconfig', '--encryption', 'repokey') + module.parse_arguments({}, '--config', 'myconfig', '--encryption', 'repokey') def test_parse_arguments_allows_encryption_mode_with_init(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey') - - -def test_parse_arguments_requires_encryption_mode_with_init(): - flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - - with pytest.raises(SystemExit): - module.parse_arguments('--config', 'myconfig', 'init') + module.parse_arguments({}, '--config', 'myconfig', 'init', '--encryption', 'repokey') def test_parse_arguments_disallows_append_only_without_init(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--config', 'myconfig', '--append-only') + module.parse_arguments({}, '--config', 'myconfig', '--append-only') def test_parse_arguments_disallows_storage_quota_without_init(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--config', 'myconfig', '--storage-quota', '5G') + module.parse_arguments({}, '--config', 'myconfig', '--storage-quota', '5G') def test_parse_arguments_allows_init_and_prune(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey', 'prune') + module.parse_arguments({}, '--config', 'myconfig', 'init', '--encryption', 'repokey', 'prune') def test_parse_arguments_allows_init_and_create(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'init', '--encryption', 'repokey', 'create') + module.parse_arguments({}, '--config', 'myconfig', 'init', '--encryption', 'repokey', 'create') def test_parse_arguments_allows_repository_with_extract(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) module.parse_arguments( - '--config', 'myconfig', 'extract', '--repository', 'test.borg', '--archive', 'test' + {}, '--config', 'myconfig', 'extract', '--repository', 'test.borg', '--archive', 'test' ) @@ -285,6 +413,7 @@ def test_parse_arguments_allows_repository_with_mount(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) module.parse_arguments( + {}, '--config', 'myconfig', 'mount', @@ -300,276 +429,247 @@ def test_parse_arguments_allows_repository_with_mount(): def test_parse_arguments_allows_repository_with_list(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'list', '--repository', 'test.borg') + module.parse_arguments({}, '--config', 'myconfig', 'list', '--repository', 'test.borg') def test_parse_arguments_disallows_archive_unless_action_consumes_it(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--config', 'myconfig', '--archive', 'test') + module.parse_arguments({}, '--config', 'myconfig', '--archive', 'test') def test_parse_arguments_disallows_paths_unless_action_consumes_it(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--config', 'myconfig', '--path', 'test') + module.parse_arguments({}, '--config', 'myconfig', '--path', 'test') def test_parse_arguments_disallows_other_actions_with_config_bootstrap(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('config', 'bootstrap', '--repository', 'test.borg', 'list') + module.parse_arguments({}, 'config', 'bootstrap', '--repository', 'test.borg', 'list') def test_parse_arguments_allows_archive_with_extract(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'extract', '--archive', 'test') + module.parse_arguments({}, '--config', 'myconfig', 'extract', '--archive', 'test') def test_parse_arguments_allows_archive_with_mount(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) module.parse_arguments( - '--config', 'myconfig', 'mount', '--archive', 'test', '--mount-point', '/mnt' + {}, '--config', 'myconfig', 'mount', '--archive', 'test', '--mount-point', '/mnt' ) def test_parse_arguments_allows_archive_with_restore(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'restore', '--archive', 'test') + module.parse_arguments({}, '--config', 'myconfig', 'restore', '--archive', 'test') def test_parse_arguments_allows_archive_with_list(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--config', 'myconfig', 'list', '--archive', 'test') + module.parse_arguments({}, '--config', 'myconfig', 'list', '--archive', 'test') def test_parse_arguments_requires_archive_with_extract(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit): - module.parse_arguments('--config', 'myconfig', 'extract') + module.parse_arguments({}, '--config', 'myconfig', 'extract') def test_parse_arguments_requires_archive_with_restore(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit): - module.parse_arguments('--config', 'myconfig', 'restore') + module.parse_arguments({}, '--config', 'myconfig', 'restore') def test_parse_arguments_requires_mount_point_with_mount(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit): - module.parse_arguments('--config', 'myconfig', 'mount', '--archive', 'test') + module.parse_arguments({}, '--config', 'myconfig', 'mount', '--archive', 'test') def test_parse_arguments_requires_mount_point_with_umount(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit): - module.parse_arguments('--config', 'myconfig', 'umount') + module.parse_arguments({}, '--config', 'myconfig', 'umount') def test_parse_arguments_allows_progress_before_create(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--progress', 'create', 'list') + module.parse_arguments({}, '--progress', 'create', 'list') def test_parse_arguments_allows_progress_after_create(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('create', '--progress', 'list') + module.parse_arguments({}, 'create', '--progress', 'list') def test_parse_arguments_allows_progress_and_extract(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--progress', 'extract', '--archive', 'test', 'list') + module.parse_arguments({}, '--progress', 'extract', '--archive', 'test', 'list') def test_parse_arguments_disallows_progress_without_create(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--progress', 'list') + module.parse_arguments({}, '--progress', 'list') def test_parse_arguments_with_stats_and_create_flags_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--stats', 'create', 'list') + module.parse_arguments({}, '--stats', 'create', 'list') def test_parse_arguments_with_stats_and_prune_flags_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--stats', 'prune', 'list') + module.parse_arguments({}, '--stats', 'prune', 'list') def test_parse_arguments_with_stats_flag_but_no_create_or_prune_flag_raises_value_error(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--stats', 'list') + module.parse_arguments({}, '--stats', 'list') def test_parse_arguments_with_list_and_create_flags_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--list', 'create') + module.parse_arguments({}, '--list', 'create') def test_parse_arguments_with_list_and_prune_flags_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--list', 'prune') + module.parse_arguments({}, '--list', 'prune') def test_parse_arguments_with_list_flag_but_no_relevant_action_raises_value_error(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - with pytest.raises(SystemExit): - module.parse_arguments('--list', 'repo-create') - - -def test_parse_arguments_disallows_list_with_progress_for_create_action(): - flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - with pytest.raises(ValueError): - module.parse_arguments('create', '--list', '--progress') - - -def test_parse_arguments_disallows_list_with_json_for_create_action(): - flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - - with pytest.raises(ValueError): - module.parse_arguments('create', '--list', '--json') + module.parse_arguments({}, '--list', 'repo-create') def test_parse_arguments_allows_json_with_list_or_info(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('list', '--json') - module.parse_arguments('info', '--json') + module.parse_arguments({}, 'list', '--json') + module.parse_arguments({}, 'info', '--json') def test_parse_arguments_disallows_json_with_both_list_and_info(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('list', 'info', '--json') + module.parse_arguments({}, 'list', 'info', '--json') def test_parse_arguments_disallows_json_with_both_list_and_repo_info(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('list', 'repo-info', '--json') + module.parse_arguments({}, 'list', 'repo-info', '--json') def test_parse_arguments_disallows_json_with_both_repo_info_and_info(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('repo-info', 'info', '--json') - - -def test_parse_arguments_disallows_transfer_with_both_archive_and_match_archives(): - flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - - with pytest.raises(ValueError): - module.parse_arguments( - 'transfer', - '--source-repository', - 'source.borg', - '--archive', - 'foo', - '--match-archives', - 'sh:*bar', - ) + module.parse_arguments({}, 'repo-info', 'info', '--json') def test_parse_arguments_disallows_list_with_both_prefix_and_match_archives(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('list', '--prefix', 'foo', '--match-archives', 'sh:*bar') + module.parse_arguments({}, 'list', '--prefix', 'foo', '--match-archives', 'sh:*bar') def test_parse_arguments_disallows_repo_list_with_both_prefix_and_match_archives(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('repo-list', '--prefix', 'foo', '--match-archives', 'sh:*bar') + module.parse_arguments({}, 'repo-list', '--prefix', 'foo', '--match-archives', 'sh:*bar') def test_parse_arguments_disallows_info_with_both_archive_and_match_archives(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('info', '--archive', 'foo', '--match-archives', 'sh:*bar') + module.parse_arguments({}, 'info', '--archive', 'foo', '--match-archives', 'sh:*bar') def test_parse_arguments_disallows_info_with_both_archive_and_prefix(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('info', '--archive', 'foo', '--prefix', 'bar') + module.parse_arguments({}, 'info', '--archive', 'foo', '--prefix', 'bar') def test_parse_arguments_disallows_info_with_both_prefix_and_match_archives(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('info', '--prefix', 'foo', '--match-archives', 'sh:*bar') + module.parse_arguments({}, 'info', '--prefix', 'foo', '--match-archives', 'sh:*bar') def test_parse_arguments_check_only_extract_does_not_raise_extract_subparser_error(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('check', '--only', 'extract') + module.parse_arguments({}, 'check', '--only', 'extract') def test_parse_arguments_extract_archive_check_does_not_raise_check_subparser_error(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('extract', '--archive', 'check') + module.parse_arguments({}, 'extract', '--archive', 'check') def test_parse_arguments_extract_with_check_only_extract_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('extract', '--archive', 'name', 'check', '--only', 'extract') + module.parse_arguments({}, 'extract', '--archive', 'name', 'check', '--only', 'extract') def test_parse_arguments_bootstrap_without_config_errors(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('bootstrap') + module.parse_arguments({}, 'bootstrap') def test_parse_arguments_config_with_no_subaction_errors(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('config') + module.parse_arguments({}, 'config') def test_parse_arguments_config_with_help_shows_config_help(capsys): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit) as exit: - module.parse_arguments('config', '--help') + module.parse_arguments({}, 'config', '--help') assert exit.value.code == 0 captured = capsys.readouterr() @@ -582,7 +682,7 @@ def test_parse_arguments_config_with_subaction_but_missing_flags_errors(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit) as exit: - module.parse_arguments('config', 'bootstrap') + module.parse_arguments({}, 'config', 'bootstrap') assert exit.value.code == 2 @@ -591,7 +691,7 @@ def test_parse_arguments_config_with_subaction_and_help_shows_subaction_help(cap flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(SystemExit) as exit: - module.parse_arguments('config', 'bootstrap', '--help') + module.parse_arguments({}, 'config', 'bootstrap', '--help') assert exit.value.code == 0 captured = capsys.readouterr() @@ -601,26 +701,30 @@ def test_parse_arguments_config_with_subaction_and_help_shows_subaction_help(cap def test_parse_arguments_config_with_subaction_and_required_flags_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('config', 'bootstrap', '--repository', 'repo.borg') + module.parse_arguments({}, 'config', 'bootstrap', '--repository', 'repo.borg') def test_parse_arguments_config_with_subaction_and_global_flags_at_start_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('--verbosity', '1', 'config', 'bootstrap', '--repository', 'repo.borg') + module.parse_arguments( + {}, '--verbosity', '1', 'config', 'bootstrap', '--repository', 'repo.borg' + ) def test_parse_arguments_config_with_subaction_and_global_flags_at_end_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('config', 'bootstrap', '--repository', 'repo.borg', '--verbosity', '1') + module.parse_arguments( + {}, 'config', 'bootstrap', '--repository', 'repo.borg', '--verbosity', '1' + ) def test_parse_arguments_config_with_subaction_and_explicit_config_file_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) module.parse_arguments( - 'config', 'bootstrap', '--repository', 'repo.borg', '--config', 'test.yaml' + {}, 'config', 'bootstrap', '--repository', 'repo.borg', '--config', 'test.yaml' ) @@ -628,10 +732,23 @@ def test_parse_arguments_with_borg_action_and_dry_run_raises(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) with pytest.raises(ValueError): - module.parse_arguments('--dry-run', 'borg', 'list') + module.parse_arguments({}, '--dry-run', 'borg', 'list') def test_parse_arguments_with_borg_action_and_no_dry_run_does_not_raise(): flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) - module.parse_arguments('borg', 'list') + module.parse_arguments({}, 'borg', 'list') + + +def test_parse_arguments_with_argument_from_schema_does_not_raise(): + flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default']) + + module.parse_arguments( + { + 'type': 'object', + 'properties': {'foo': {'type': 'object', 'properties': {'bar': {'type': 'integer'}}}}, + }, + '--foo.bar', + '3', + ) diff --git a/tests/integration/config/test_arguments.py b/tests/integration/config/test_arguments.py new file mode 100644 index 00000000..9bcc37ff --- /dev/null +++ b/tests/integration/config/test_arguments.py @@ -0,0 +1,34 @@ +import pytest + +from borgmatic.config import arguments as module + + +def test_convert_value_type_passes_through_non_string_value(): + assert module.convert_value_type([1, 2], 'array') == [1, 2] + + +def test_convert_value_type_passes_through_string_option_type(): + assert module.convert_value_type('foo', 'string') == 'foo' + + +def test_convert_value_type_parses_array_option_type(): + assert module.convert_value_type('[foo, bar]', 'array') == ['foo', 'bar'] + + +def test_convert_value_type_with_array_option_type_and_no_array_raises(): + with pytest.raises(ValueError): + module.convert_value_type('{foo, bar}', 'array') + + +def test_convert_value_type_parses_object_option_type(): + assert module.convert_value_type('{foo: bar}', 'object') == {'foo': 'bar'} + + +def test_convert_value_type_with_invalid_value_raises(): + with pytest.raises(ValueError): + module.convert_value_type('{foo, bar', 'object') + + +def test_convert_value_type_with_unknown_option_type_raises(): + with pytest.raises(ValueError): + module.convert_value_type('{foo, bar}', 'thingy') diff --git a/tests/integration/config/test_generate.py b/tests/integration/config/test_generate.py index 0781e8d1..71c0abc5 100644 --- a/tests/integration/config/test_generate.py +++ b/tests/integration/config/test_generate.py @@ -16,6 +16,116 @@ def test_insert_newline_before_comment_does_not_raise(): module.insert_newline_before_comment(config, field_name) +def test_schema_to_sample_configuration_comments_out_non_default_options(): + schema = { + 'type': 'object', + 'properties': dict( + [ + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ('source_directories', {'type': 'string', 'example': 'Example 3'}), + ] + ), + } + + config = module.schema_to_sample_configuration(schema) + + assert config == dict( + [ + ('field1', 'Example 1'), + ('field2', 'Example 2'), + ('source_directories', 'Example 3'), + ] + ) + assert 'COMMENT_OUT' in config.ca.items['field1'][1][-1]._value + assert 'COMMENT_OUT' in config.ca.items['field2'][1][-1]._value + assert 'source_directories' not in config.ca.items + + +def test_schema_to_sample_configuration_comments_out_non_source_config_options(): + schema = { + 'type': 'object', + 'properties': dict( + [ + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ('field3', {'type': 'string', 'example': 'Example 3'}), + ] + ), + } + source_config = {'field3': 'value'} + + config = module.schema_to_sample_configuration(schema, source_config) + + assert config == dict( + [ + ('field1', 'Example 1'), + ('field2', 'Example 2'), + ('field3', 'Example 3'), + ] + ) + assert 'COMMENT_OUT' in config.ca.items['field1'][1][-1]._value + assert 'COMMENT_OUT' in config.ca.items['field2'][1][-1]._value + assert 'field3' not in config.ca.items + + +def test_schema_to_sample_configuration_comments_out_non_default_options_in_sequence_of_maps(): + schema = { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': dict( + [ + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ('source_directories', {'type': 'string', 'example': 'Example 3'}), + ] + ), + }, + } + + config = module.schema_to_sample_configuration(schema) + + assert config == [ + dict( + [('field1', 'Example 1'), ('field2', 'Example 2'), ('source_directories', 'Example 3')] + ) + ] + + # The first field in a sequence does not get commented. + assert 'field1' not in config[0].ca.items + assert 'COMMENT_OUT' in config[0].ca.items['field2'][1][-1]._value + assert 'source_directories' not in config[0].ca.items + + +def test_schema_to_sample_configuration_comments_out_non_source_config_options_in_sequence_of_maps(): + schema = { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': dict( + [ + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ('field3', {'type': 'string', 'example': 'Example 3'}), + ] + ), + }, + } + source_config = [{'field3': 'value'}] + + config = module.schema_to_sample_configuration(schema, source_config) + + assert config == [ + dict([('field1', 'Example 1'), ('field2', 'Example 2'), ('field3', 'Example 3')]) + ] + + # The first field in a sequence does not get commented. + assert 'field1' not in config[0].ca.items + assert 'COMMENT_OUT' in config[0].ca.items['field2'][1][-1]._value + assert 'field3' not in config[0].ca.items + + def test_comment_out_line_skips_blank_line(): line = ' \n' @@ -152,7 +262,7 @@ def test_add_comments_to_configuration_sequence_of_maps_without_description_does module.add_comments_to_configuration_sequence(config, schema) -def test_add_comments_to_configuration_object_does_not_raise(): +def test_add_comments_to_configuration_comments_out_non_default_options(): # Ensure that it can deal with fields both in the schema and missing from the schema. config = module.ruamel.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)]) schema = { @@ -162,44 +272,43 @@ def test_add_comments_to_configuration_object_does_not_raise(): module.add_comments_to_configuration_object(config, schema) + assert 'COMMENT_OUT' in config.ca.items['foo'][1][-1]._value + assert 'COMMENT_OUT' in config.ca.items['bar'][1][-1]._value + assert 'baz' not in config.ca.items -def test_add_comments_to_configuration_object_with_skip_first_does_not_raise(): - config = module.ruamel.yaml.comments.CommentedMap([('foo', 33)]) - schema = {'type': 'object', 'properties': {'foo': {'description': 'Foo'}}} + +def test_add_comments_to_configuration_comments_out_non_source_config_options(): + # Ensure that it can deal with fields both in the schema and missing from the schema. + config = module.ruamel.yaml.comments.CommentedMap( + [('repositories', 33), ('bar', 44), ('baz', 55)] + ) + schema = { + 'type': 'object', + 'properties': { + 'repositories': {'description': 'repositories'}, + 'bar': {'description': 'Bar'}, + }, + } + + module.add_comments_to_configuration_object(config, schema) + + assert 'repositories' in config.ca.items + assert 'COMMENT_OUT' in config.ca.items['bar'][1][-1]._value + assert 'baz' not in config.ca.items + + +def test_add_comments_to_configuration_object_with_skip_first_does_not_comment_out_first_option(): + config = module.ruamel.yaml.comments.CommentedMap([('foo', 33), ('bar', 44), ('baz', 55)]) + schema = { + 'type': 'object', + 'properties': {'foo': {'description': 'Foo'}, 'bar': {'description': 'Bar'}}, + } module.add_comments_to_configuration_object(config, schema, skip_first=True) - -def test_remove_commented_out_sentinel_keeps_other_comments(): - field_name = 'foo' - config = module.ruamel.yaml.comments.CommentedMap([(field_name, 33)]) - config.yaml_set_comment_before_after_key(key=field_name, before='Actual comment.\nCOMMENT_OUT') - - module.remove_commented_out_sentinel(config, field_name) - - comments = config.ca.items[field_name][module.RUAMEL_YAML_COMMENTS_INDEX] - assert len(comments) == 1 - assert comments[0].value == '# Actual comment.\n' - - -def test_remove_commented_out_sentinel_without_sentinel_keeps_other_comments(): - field_name = 'foo' - config = module.ruamel.yaml.comments.CommentedMap([(field_name, 33)]) - config.yaml_set_comment_before_after_key(key=field_name, before='Actual comment.') - - module.remove_commented_out_sentinel(config, field_name) - - comments = config.ca.items[field_name][module.RUAMEL_YAML_COMMENTS_INDEX] - assert len(comments) == 1 - assert comments[0].value == '# Actual comment.\n' - - -def test_remove_commented_out_sentinel_on_unknown_field_does_not_raise(): - field_name = 'foo' - config = module.ruamel.yaml.comments.CommentedMap([(field_name, 33)]) - config.yaml_set_comment_before_after_key(key=field_name, before='Actual comment.') - - module.remove_commented_out_sentinel(config, 'unknown') + assert 'foo' not in config.ca.items + assert 'COMMENT_OUT' in config.ca.items['bar'][1][-1]._value + assert 'baz' not in config.ca.items def test_generate_sample_configuration_does_not_raise(): diff --git a/tests/integration/config/test_schema.py b/tests/integration/config/test_schema.py index 30eb1c47..5cf114b1 100644 --- a/tests/integration/config/test_schema.py +++ b/tests/integration/config/test_schema.py @@ -14,7 +14,13 @@ def test_schema_line_length_stays_under_limit(): assert len(line.rstrip('\n')) <= MAXIMUM_LINE_LENGTH -ACTIONS_MODULE_NAMES_TO_OMIT = {'arguments', 'change_passphrase', 'export_key', 'json'} +ACTIONS_MODULE_NAMES_TO_OMIT = { + 'arguments', + 'change_passphrase', + 'export_key', + 'import_key', + 'json', +} ACTIONS_MODULE_NAMES_TO_ADD = {'key', 'umount'} diff --git a/tests/integration/config/test_validate.py b/tests/integration/config/test_validate.py index 9cd5c980..e74989c6 100644 --- a/tests/integration/config/test_validate.py +++ b/tests/integration/config/test_validate.py @@ -58,7 +58,9 @@ def test_parse_configuration_transforms_file_into_mapping(): ''' ) - config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + config, config_paths, logs = module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} + ) assert config == { 'source_directories': ['/home', '/etc'], @@ -86,7 +88,9 @@ def test_parse_configuration_passes_through_quoted_punctuation(): ''' ) - config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + config, config_paths, logs = module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} + ) assert config == { 'source_directories': [f'/home/{string.punctuation}'], @@ -119,7 +123,9 @@ def test_parse_configuration_with_schema_lacking_examples_does_not_raise(): ''', ) - module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} + ) def test_parse_configuration_inlines_include_inside_deprecated_section(): @@ -145,7 +151,9 @@ def test_parse_configuration_inlines_include_inside_deprecated_section(): include_file.name = 'include.yaml' builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) - config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + config, config_paths, logs = module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} + ) assert config == { 'source_directories': ['/home'], @@ -181,7 +189,9 @@ def test_parse_configuration_merges_include(): include_file.name = 'include.yaml' builtins.should_receive('open').with_args('/tmp/include.yaml').and_return(include_file) - config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + config, config_paths, logs = module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} + ) assert config == { 'source_directories': ['/home'], @@ -196,7 +206,9 @@ def test_parse_configuration_merges_include(): def test_parse_configuration_raises_for_missing_config_file(): with pytest.raises(FileNotFoundError): - module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} + ) def test_parse_configuration_raises_for_missing_schema_file(): @@ -208,14 +220,18 @@ def test_parse_configuration_raises_for_missing_schema_file(): builtins.should_receive('open').with_args('/tmp/schema.yaml').and_raise(FileNotFoundError) with pytest.raises(FileNotFoundError): - module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} + ) def test_parse_configuration_raises_for_syntax_error(): mock_config_and_schema('foo:\nbar') with pytest.raises(ValueError): - module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} + ) def test_parse_configuration_raises_for_validation_error(): @@ -228,7 +244,9 @@ def test_parse_configuration_raises_for_validation_error(): ) with pytest.raises(module.Validation_error): - module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} + ) def test_parse_configuration_applies_overrides(): @@ -245,7 +263,10 @@ def test_parse_configuration_applies_overrides(): ) config, config_paths, logs = module.parse_configuration( - '/tmp/config.yaml', '/tmp/schema.yaml', overrides=['local_path=borg2'] + '/tmp/config.yaml', + '/tmp/schema.yaml', + arguments={'global': flexmock()}, + overrides=['local_path=borg2'], ) assert config == { @@ -273,7 +294,9 @@ def test_parse_configuration_applies_normalization_after_environment_variable_in ) flexmock(os).should_receive('getenv').replace_with(lambda variable_name, default: default) - config, config_paths, logs = module.parse_configuration('/tmp/config.yaml', '/tmp/schema.yaml') + config, config_paths, logs = module.parse_configuration( + '/tmp/config.yaml', '/tmp/schema.yaml', arguments={'global': flexmock()} + ) assert config == { 'source_directories': ['/home'], diff --git a/tests/unit/actions/config/test_bootstrap.py b/tests/unit/actions/config/test_bootstrap.py index 7a21e715..ece91197 100644 --- a/tests/unit/actions/config/test_bootstrap.py +++ b/tests/unit/actions/config/test_bootstrap.py @@ -105,7 +105,7 @@ def test_get_config_paths_translates_ssh_command_argument_to_config(): flexmock(module.borgmatic.config.paths).should_receive( 'get_borgmatic_source_directory' ).and_return('/source') - config = flexmock() + config = {} flexmock(module).should_receive('make_bootstrap_config').and_return(config) bootstrap_arguments = flexmock( repository='repo', @@ -267,11 +267,11 @@ def test_run_bootstrap_does_not_raise(): archive='archive', destination='dest', strip_components=1, - progress=False, user_runtime_directory='/borgmatic', ssh_command=None, local_path='borg7', remote_path='borg8', + progress=None, ) global_arguments = flexmock( dry_run=False, @@ -299,7 +299,7 @@ def test_run_bootstrap_does_not_raise(): def test_run_bootstrap_translates_ssh_command_argument_to_config(): - config = flexmock() + config = {} flexmock(module).should_receive('make_bootstrap_config').and_return(config) flexmock(module).should_receive('get_config_paths').and_return(['/borgmatic/config.yaml']) bootstrap_arguments = flexmock( @@ -307,11 +307,11 @@ def test_run_bootstrap_translates_ssh_command_argument_to_config(): archive='archive', destination='dest', strip_components=1, - progress=False, user_runtime_directory='/borgmatic', ssh_command='ssh -i key', local_path='borg7', remote_path='borg8', + progress=None, ) global_arguments = flexmock( dry_run=False, @@ -333,13 +333,12 @@ def test_run_bootstrap_translates_ssh_command_argument_to_config(): 'repo', 'archive', object, - config, + {'progress': False}, object, object, extract_to_stdout=False, destination_path='dest', strip_components=1, - progress=False, local_path='borg7', remote_path='borg8', ).and_return(extract_process).once() diff --git a/tests/unit/actions/test_check.py b/tests/unit/actions/test_check.py index 90b716b5..e7eee5c4 100644 --- a/tests/unit/actions/test_check.py +++ b/tests/unit/actions/test_check.py @@ -577,7 +577,6 @@ def test_collect_spot_check_source_paths_parses_borg_output(): borgmatic_runtime_directory='/run/borgmatic', local_path=object, remote_path=object, - list_files=True, stream_processes=True, ).and_return((('borg', 'create'), ('repo::archive',), flexmock())) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -625,7 +624,6 @@ def test_collect_spot_check_source_paths_passes_through_stream_processes_false() borgmatic_runtime_directory='/run/borgmatic', local_path=object, remote_path=object, - list_files=True, stream_processes=False, ).and_return((('borg', 'create'), ('repo::archive',), flexmock())) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -673,7 +671,6 @@ def test_collect_spot_check_source_paths_without_working_directory_parses_borg_o borgmatic_runtime_directory='/run/borgmatic', local_path=object, remote_path=object, - list_files=True, stream_processes=True, ).and_return((('borg', 'create'), ('repo::archive',), flexmock())) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -721,7 +718,6 @@ def test_collect_spot_check_source_paths_skips_directories(): borgmatic_runtime_directory='/run/borgmatic', local_path=object, remote_path=object, - list_files=True, stream_processes=True, ).and_return((('borg', 'create'), ('repo::archive',), flexmock())) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -860,14 +856,13 @@ def test_collect_spot_check_source_paths_uses_working_directory(): flexmock(module.borgmatic.borg.create).should_receive('make_base_create_command').with_args( dry_run=True, repository_path='repo', - config=object, + config={'working_directory': '/working/dir', 'list_details': True}, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version=object, global_arguments=object, borgmatic_runtime_directory='/run/borgmatic', local_path=object, remote_path=object, - list_files=True, stream_processes=True, ).and_return((('borg', 'create'), ('repo::archive',), flexmock())) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -903,6 +898,7 @@ def test_compare_spot_check_hashes_returns_paths_having_failing_hashes(): None, ) flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.os.path).should_receive('islink').and_return(False) flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return( @@ -943,6 +939,7 @@ def test_compare_spot_check_hashes_returns_relative_paths_having_failing_hashes( None, ) flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.os.path).should_receive('islink').and_return(False) flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).with_args(('xxh64sum', 'foo', 'bar'), working_directory=None).and_return( @@ -983,6 +980,7 @@ def test_compare_spot_check_hashes_handles_data_sample_percentage_above_100(): None, ) flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.os.path).should_receive('islink').and_return(False) flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return( @@ -1023,6 +1021,7 @@ def test_compare_spot_check_hashes_uses_xxh64sum_command_option(): None, ) flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.os.path).should_receive('islink').and_return(False) flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).with_args(('/usr/local/bin/xxh64sum', '/foo', '/bar'), working_directory=None).and_return( @@ -1060,6 +1059,7 @@ def test_compare_spot_check_hashes_considers_path_missing_from_archive_as_not_ma None, ) flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.os.path).should_receive('islink').and_return(False) flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return( @@ -1088,6 +1088,42 @@ def test_compare_spot_check_hashes_considers_path_missing_from_archive_as_not_ma ) == ('/bar',) +def test_compare_spot_check_hashes_considers_symlink_path_as_not_matching(): + flexmock(module.random).should_receive('sample').replace_with( + lambda population, count: population[:count] + ) + flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return( + None, + ) + flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.os.path).should_receive('islink').with_args('/foo').and_return(False) + flexmock(module.os.path).should_receive('islink').with_args('/bar').and_return(True) + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).with_args(('xxh64sum', '/foo'), working_directory=None).and_return('hash1 /foo') + flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return( + ['hash1 foo', 'hash2 bar'] + ) + + assert module.compare_spot_check_hashes( + repository={'path': 'repo'}, + archive='archive', + config={ + 'checks': [ + { + 'name': 'spot', + 'data_sample_percentage': 50, + }, + ] + }, + local_borg_version=flexmock(), + global_arguments=flexmock(), + local_path=flexmock(), + remote_path=flexmock(), + source_paths=('/foo', '/bar', '/baz', '/quux'), + ) == ('/bar',) + + def test_compare_spot_check_hashes_considers_non_existent_path_as_not_matching(): flexmock(module.random).should_receive('sample').replace_with( lambda population, count: population[:count] @@ -1097,6 +1133,7 @@ def test_compare_spot_check_hashes_considers_non_existent_path_as_not_matching() ) flexmock(module.os.path).should_receive('exists').with_args('/foo').and_return(True) flexmock(module.os.path).should_receive('exists').with_args('/bar').and_return(False) + flexmock(module.os.path).should_receive('islink').and_return(False) flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).with_args(('xxh64sum', '/foo'), working_directory=None).and_return('hash1 /foo') @@ -1132,6 +1169,7 @@ def test_compare_spot_check_hashes_with_too_many_paths_feeds_them_to_commands_in None, ) flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.os.path).should_receive('islink').and_return(False) flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return( @@ -1178,6 +1216,7 @@ def test_compare_spot_check_hashes_uses_working_directory_to_access_source_paths ) flexmock(module.os.path).should_receive('exists').with_args('/working/dir/foo').and_return(True) flexmock(module.os.path).should_receive('exists').with_args('/working/dir/bar').and_return(True) + flexmock(module.os.path).should_receive('islink').and_return(False) flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).with_args(('xxh64sum', 'foo', 'bar'), working_directory='/working/dir').and_return( @@ -1405,7 +1444,6 @@ def test_run_check_checks_archives_for_configured_repository(): flexmock(module).should_receive('make_check_time_path') flexmock(module).should_receive('write_check_time') flexmock(module.borgmatic.borg.extract).should_receive('extract_last_archive_dry_run').never() - flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) check_arguments = flexmock( repository=None, progress=flexmock(), @@ -1419,7 +1457,6 @@ def test_run_check_checks_archives_for_configured_repository(): config_filename='test.yaml', repository={'path': 'repo'}, config={'repositories': ['repo']}, - hook_context={}, local_borg_version=None, check_arguments=check_arguments, global_arguments=global_arguments, @@ -1441,7 +1478,6 @@ def test_run_check_runs_configured_extract_check(): flexmock(module.borgmatic.borg.extract).should_receive('extract_last_archive_dry_run').once() flexmock(module).should_receive('make_check_time_path') flexmock(module).should_receive('write_check_time') - flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) check_arguments = flexmock( repository=None, progress=flexmock(), @@ -1455,7 +1491,6 @@ def test_run_check_runs_configured_extract_check(): config_filename='test.yaml', repository={'path': 'repo'}, config={'repositories': ['repo']}, - hook_context={}, local_borg_version=None, check_arguments=check_arguments, global_arguments=global_arguments, @@ -1480,7 +1515,6 @@ def test_run_check_runs_configured_spot_check(): flexmock(module.borgmatic.actions.check).should_receive('spot_check').once() flexmock(module).should_receive('make_check_time_path') flexmock(module).should_receive('write_check_time') - flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) check_arguments = flexmock( repository=None, progress=flexmock(), @@ -1494,7 +1528,6 @@ def test_run_check_runs_configured_spot_check(): config_filename='test.yaml', repository={'path': 'repo'}, config={'repositories': ['repo']}, - hook_context={}, local_borg_version=None, check_arguments=check_arguments, global_arguments=global_arguments, @@ -1516,7 +1549,6 @@ def test_run_check_without_checks_runs_nothing_except_hooks(): flexmock(module).should_receive('make_check_time_path') flexmock(module).should_receive('write_check_time').never() flexmock(module.borgmatic.borg.extract).should_receive('extract_last_archive_dry_run').never() - flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) check_arguments = flexmock( repository=None, progress=flexmock(), @@ -1530,7 +1562,6 @@ def test_run_check_without_checks_runs_nothing_except_hooks(): config_filename='test.yaml', repository={'path': 'repo'}, config={'repositories': ['repo']}, - hook_context={}, local_borg_version=None, check_arguments=check_arguments, global_arguments=global_arguments, @@ -1569,7 +1600,6 @@ def test_run_check_checks_archives_in_selected_repository(): config_filename='test.yaml', repository={'path': 'repo'}, config={'repositories': ['repo']}, - hook_context={}, local_borg_version=None, check_arguments=check_arguments, global_arguments=global_arguments, @@ -1597,7 +1627,6 @@ def test_run_check_bails_if_repository_does_not_match(): config_filename='test.yaml', repository={'path': 'repo'}, config={'repositories': ['repo']}, - hook_context={}, local_borg_version=None, check_arguments=check_arguments, global_arguments=global_arguments, diff --git a/tests/unit/actions/test_compact.py b/tests/unit/actions/test_compact.py index 0df83fdd..04a3a25e 100644 --- a/tests/unit/actions/test_compact.py +++ b/tests/unit/actions/test_compact.py @@ -8,9 +8,11 @@ def test_compact_actions_calls_hooks_for_configured_repository(): flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never() flexmock(module.borgmatic.borg.compact).should_receive('compact_segments').once() - flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) compact_arguments = flexmock( - repository=None, progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock() + repository=None, + progress=flexmock(), + cleanup_commits=flexmock(), + compact_threshold=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -18,7 +20,6 @@ def test_compact_actions_calls_hooks_for_configured_repository(): config_filename='test.yaml', repository={'path': 'repo'}, config={}, - hook_context={}, local_borg_version=None, compact_arguments=compact_arguments, global_arguments=global_arguments, @@ -36,7 +37,10 @@ def test_compact_runs_with_selected_repository(): flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.compact).should_receive('compact_segments').once() compact_arguments = flexmock( - repository=flexmock(), progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock() + repository=flexmock(), + progress=flexmock(), + cleanup_commits=flexmock(), + compact_threshold=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -44,7 +48,6 @@ def test_compact_runs_with_selected_repository(): config_filename='test.yaml', repository={'path': 'repo'}, config={}, - hook_context={}, local_borg_version=None, compact_arguments=compact_arguments, global_arguments=global_arguments, @@ -62,7 +65,10 @@ def test_compact_bails_if_repository_does_not_match(): ).once().and_return(False) flexmock(module.borgmatic.borg.compact).should_receive('compact_segments').never() compact_arguments = flexmock( - repository=flexmock(), progress=flexmock(), cleanup_commits=flexmock(), threshold=flexmock() + repository=flexmock(), + progress=flexmock(), + cleanup_commits=flexmock(), + compact_threshold=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -70,7 +76,6 @@ def test_compact_bails_if_repository_does_not_match(): config_filename='test.yaml', repository={'path': 'repo'}, config={}, - hook_context={}, local_borg_version=None, compact_arguments=compact_arguments, global_arguments=global_arguments, diff --git a/tests/unit/actions/test_create.py b/tests/unit/actions/test_create.py index 8ba1eed0..6761f8f4 100644 --- a/tests/unit/actions/test_create.py +++ b/tests/unit/actions/test_create.py @@ -5,16 +5,21 @@ import pytest from flexmock import flexmock from borgmatic.actions import create as module -from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type +from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type @pytest.mark.parametrize( 'pattern_line,expected_pattern', ( - ('R /foo', Pattern('/foo')), - ('P sh', Pattern('sh', Pattern_type.PATTERN_STYLE)), - ('+ /foo*', Pattern('/foo*', Pattern_type.INCLUDE)), - ('+ sh:/foo*', Pattern('/foo*', Pattern_type.INCLUDE, Pattern_style.SHELL)), + ('R /foo', Pattern('/foo', source=Pattern_source.CONFIG)), + ('P sh', Pattern('sh', Pattern_type.PATTERN_STYLE, source=Pattern_source.CONFIG)), + ('+ /foo*', Pattern('/foo*', Pattern_type.INCLUDE, source=Pattern_source.CONFIG)), + ( + '+ sh:/foo*', + Pattern( + '/foo*', Pattern_type.INCLUDE, Pattern_style.SHELL, source=Pattern_source.CONFIG + ), + ), ), ) def test_parse_pattern_transforms_pattern_line_to_instance(pattern_line, expected_pattern): @@ -28,8 +33,8 @@ def test_parse_pattern_with_invalid_pattern_line_errors(): def test_collect_patterns_converts_source_directories(): assert module.collect_patterns({'source_directories': ['/foo', '/bar']}) == ( - Pattern('/foo'), - Pattern('/bar'), + Pattern('/foo', source=Pattern_source.CONFIG), + Pattern('/bar', source=Pattern_source.CONFIG), ) @@ -48,9 +53,15 @@ def test_collect_patterns_parses_config_patterns(): def test_collect_patterns_converts_exclude_patterns(): assert module.collect_patterns({'exclude_patterns': ['/foo', '/bar', 'sh:**/baz']}) == ( - Pattern('/foo', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH), - Pattern('/bar', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH), - Pattern('**/baz', Pattern_type.NO_RECURSE, Pattern_style.SHELL), + Pattern( + '/foo', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, source=Pattern_source.CONFIG + ), + Pattern( + '/bar', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, source=Pattern_source.CONFIG + ), + Pattern( + '**/baz', Pattern_type.NO_RECURSE, Pattern_style.SHELL, source=Pattern_source.CONFIG + ), ) @@ -214,15 +225,24 @@ def test_expand_patterns_considers_none_as_no_patterns(): assert module.expand_patterns(None) == () -def test_expand_patterns_only_considers_root_patterns(): - flexmock(module).should_receive('expand_directory').with_args('~/foo', None).and_return( - ['/root/foo'] +def test_expand_patterns_expands_tildes_and_globs_in_root_patterns(): + flexmock(module.os.path).should_receive('expanduser').never() + flexmock(module).should_receive('expand_directory').and_return( + ['/root/foo/one', '/root/foo/two'] ) - flexmock(module).should_receive('expand_directory').with_args('bar*', None).never() - paths = module.expand_patterns((Pattern('~/foo'), Pattern('bar*', Pattern_type.INCLUDE))) + paths = module.expand_patterns((Pattern('~/foo/*'),)) - assert paths == (Pattern('/root/foo'), Pattern('bar*', Pattern_type.INCLUDE)) + assert paths == (Pattern('/root/foo/one'), Pattern('/root/foo/two')) + + +def test_expand_patterns_expands_only_tildes_in_non_root_patterns(): + flexmock(module).should_receive('expand_directory').never() + flexmock(module.os.path).should_receive('expanduser').and_return('/root/bar/*') + + paths = module.expand_patterns((Pattern('~/bar/*', Pattern_type.INCLUDE),)) + + assert paths == (Pattern('/root/bar/*', Pattern_type.INCLUDE),) def test_device_map_patterns_gives_device_id_per_path(): @@ -413,7 +433,6 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository(): flexmock() ) flexmock(module.borgmatic.borg.create).should_receive('create_archive').once() - flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({}) flexmock(module.borgmatic.hooks.dispatch).should_receive( 'call_hooks_even_if_unconfigured' @@ -424,9 +443,9 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository(): create_arguments = flexmock( repository=None, progress=flexmock(), - stats=flexmock(), + statistics=flexmock(), json=False, - list_files=flexmock(), + list_details=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -436,7 +455,6 @@ def test_run_create_executes_and_calls_hooks_for_configured_repository(): repository={'path': 'repo'}, config={}, config_paths=['/tmp/test.yaml'], - hook_context={}, local_borg_version=None, create_arguments=create_arguments, global_arguments=global_arguments, @@ -456,7 +474,6 @@ def test_run_create_runs_with_selected_repository(): flexmock() ) flexmock(module.borgmatic.borg.create).should_receive('create_archive').once() - flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({}) flexmock(module.borgmatic.hooks.dispatch).should_receive( 'call_hooks_even_if_unconfigured' @@ -467,9 +484,9 @@ def test_run_create_runs_with_selected_repository(): create_arguments = flexmock( repository=flexmock(), progress=flexmock(), - stats=flexmock(), + statistics=flexmock(), json=False, - list_files=flexmock(), + list_details=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -479,7 +496,6 @@ def test_run_create_runs_with_selected_repository(): repository={'path': 'repo'}, config={}, config_paths=['/tmp/test.yaml'], - hook_context={}, local_borg_version=None, create_arguments=create_arguments, global_arguments=global_arguments, @@ -500,9 +516,9 @@ def test_run_create_bails_if_repository_does_not_match(): create_arguments = flexmock( repository=flexmock(), progress=flexmock(), - stats=flexmock(), + statistics=flexmock(), json=False, - list_files=flexmock(), + list_details=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -512,7 +528,6 @@ def test_run_create_bails_if_repository_does_not_match(): repository='repo', config={}, config_paths=['/tmp/test.yaml'], - hook_context={}, local_borg_version=None, create_arguments=create_arguments, global_arguments=global_arguments, @@ -523,6 +538,72 @@ def test_run_create_bails_if_repository_does_not_match(): ) +def test_run_create_with_both_list_and_json_errors(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive( + 'repositories_match' + ).once().and_return(True) + flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').never() + flexmock(module.borgmatic.borg.create).should_receive('create_archive').never() + create_arguments = flexmock( + repository=flexmock(), + progress=flexmock(), + statistics=flexmock(), + json=True, + list_details=flexmock(), + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + with pytest.raises(ValueError): + list( + module.run_create( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={'list_details': True}, + config_paths=['/tmp/test.yaml'], + local_borg_version=None, + create_arguments=create_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + ) + + +def test_run_create_with_both_list_and_progress_errors(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive( + 'repositories_match' + ).once().and_return(True) + flexmock(module.borgmatic.config.paths).should_receive('Runtime_directory').never() + flexmock(module.borgmatic.borg.create).should_receive('create_archive').never() + create_arguments = flexmock( + repository=flexmock(), + progress=flexmock(), + statistics=flexmock(), + json=False, + list_details=flexmock(), + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + with pytest.raises(ValueError): + list( + module.run_create( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={'list_details': True, 'progress': True}, + config_paths=['/tmp/test.yaml'], + local_borg_version=None, + create_arguments=create_arguments, + global_arguments=global_arguments, + dry_run_label='', + local_path=None, + remote_path=None, + ) + ) + + def test_run_create_produces_json(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive( @@ -536,7 +617,6 @@ def test_run_create_produces_json(): ) parsed_json = flexmock() flexmock(module.borgmatic.actions.json).should_receive('parse_json').and_return(parsed_json) - flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').and_return({}) flexmock(module.borgmatic.hooks.dispatch).should_receive( 'call_hooks_even_if_unconfigured' @@ -547,9 +627,9 @@ def test_run_create_produces_json(): create_arguments = flexmock( repository=flexmock(), progress=flexmock(), - stats=flexmock(), + statistics=flexmock(), json=True, - list_files=flexmock(), + list_details=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -559,7 +639,6 @@ def test_run_create_produces_json(): repository={'path': 'repo'}, config={}, config_paths=['/tmp/test.yaml'], - hook_context={}, local_borg_version=None, create_arguments=create_arguments, global_arguments=global_arguments, diff --git a/tests/unit/actions/test_export_tar.py b/tests/unit/actions/test_export_tar.py index aea54af3..d32aa6fe 100644 --- a/tests/unit/actions/test_export_tar.py +++ b/tests/unit/actions/test_export_tar.py @@ -13,7 +13,7 @@ def test_run_export_tar_does_not_raise(): paths=flexmock(), destination=flexmock(), tar_filter=flexmock(), - list_files=flexmock(), + list_details=flexmock(), strip_components=flexmock(), ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) @@ -27,3 +27,81 @@ def test_run_export_tar_does_not_raise(): local_path=None, remote_path=None, ) + + +def test_run_export_tar_favors_flags_over_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.export_tar).should_receive('export_tar_archive').with_args( + object, + object, + object, + object, + object, + object, + object, + object, + local_path=object, + remote_path=object, + tar_filter=object, + strip_components=object, + ).once() + export_tar_arguments = flexmock( + repository=flexmock(), + archive=flexmock(), + paths=flexmock(), + destination=flexmock(), + tar_filter=flexmock(), + list_details=False, + strip_components=flexmock(), + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_export_tar( + repository={'path': 'repo'}, + config={'list_details': True}, + local_borg_version=None, + export_tar_arguments=export_tar_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) + + +def test_run_export_tar_defaults_to_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.export_tar).should_receive('export_tar_archive').with_args( + object, + object, + object, + object, + object, + object, + object, + object, + local_path=object, + remote_path=object, + tar_filter=object, + strip_components=object, + ).once() + export_tar_arguments = flexmock( + repository=flexmock(), + archive=flexmock(), + paths=flexmock(), + destination=flexmock(), + tar_filter=flexmock(), + list_details=None, + strip_components=flexmock(), + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_export_tar( + repository={'path': 'repo'}, + config={'list_details': True}, + local_borg_version=None, + export_tar_arguments=export_tar_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/actions/test_extract.py b/tests/unit/actions/test_extract.py index 7fadf4d7..1504d4ce 100644 --- a/tests/unit/actions/test_extract.py +++ b/tests/unit/actions/test_extract.py @@ -7,7 +7,6 @@ def test_run_extract_calls_hooks(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) flexmock(module.borgmatic.borg.extract).should_receive('extract_archive') - flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) extract_arguments = flexmock( paths=flexmock(), progress=flexmock(), @@ -22,7 +21,82 @@ def test_run_extract_calls_hooks(): config_filename='test.yaml', repository={'path': 'repo'}, config={'repositories': ['repo']}, - hook_context={}, + local_borg_version=None, + extract_arguments=extract_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) + + +def test_run_extract_favors_flags_over_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').with_args( + object, + object, + object, + object, + object, + object, + object, + local_path=object, + remote_path=object, + destination_path=object, + strip_components=object, + ).once() + extract_arguments = flexmock( + paths=flexmock(), + progress=False, + destination=flexmock(), + strip_components=flexmock(), + archive=flexmock(), + repository='repo', + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_extract( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={'repositories': ['repo'], 'progress': True}, + local_borg_version=None, + extract_arguments=extract_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) + + +def test_run_extract_defaults_to_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.extract).should_receive('extract_archive').with_args( + object, + object, + object, + object, + object, + object, + object, + local_path=object, + remote_path=object, + destination_path=object, + strip_components=object, + ).once() + extract_arguments = flexmock( + paths=flexmock(), + progress=None, + destination=flexmock(), + strip_components=flexmock(), + archive=flexmock(), + repository='repo', + ) + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + module.run_extract( + config_filename='test.yaml', + repository={'path': 'repo'}, + config={'repositories': ['repo'], 'progress': True}, local_borg_version=None, extract_arguments=extract_arguments, global_arguments=global_arguments, diff --git a/tests/unit/actions/test_import_key.py b/tests/unit/actions/test_import_key.py new file mode 100644 index 00000000..565e3df3 --- /dev/null +++ b/tests/unit/actions/test_import_key.py @@ -0,0 +1,20 @@ +from flexmock import flexmock + +from borgmatic.actions import import_key as module + + +def test_run_import_key_does_not_raise(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.import_key).should_receive('import_key') + import_arguments = flexmock(repository=flexmock()) + + module.run_import_key( + repository={'path': 'repo'}, + config={}, + local_borg_version=None, + import_arguments=import_arguments, + global_arguments=flexmock(), + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/actions/test_prune.py b/tests/unit/actions/test_prune.py index d5dd182e..b37e50fb 100644 --- a/tests/unit/actions/test_prune.py +++ b/tests/unit/actions/test_prune.py @@ -7,15 +7,13 @@ def test_run_prune_calls_hooks_for_configured_repository(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive('repositories_match').never() flexmock(module.borgmatic.borg.prune).should_receive('prune_archives').once() - flexmock(module.borgmatic.hooks.command).should_receive('execute_hook').times(2) - prune_arguments = flexmock(repository=None, stats=flexmock(), list_archives=flexmock()) + prune_arguments = flexmock(repository=None, statistics=flexmock(), list_details=flexmock()) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_prune( config_filename='test.yaml', repository={'path': 'repo'}, config={}, - hook_context={}, local_borg_version=None, prune_arguments=prune_arguments, global_arguments=global_arguments, @@ -31,14 +29,15 @@ def test_run_prune_runs_with_selected_repository(): 'repositories_match' ).once().and_return(True) flexmock(module.borgmatic.borg.prune).should_receive('prune_archives').once() - prune_arguments = flexmock(repository=flexmock(), stats=flexmock(), list_archives=flexmock()) + prune_arguments = flexmock( + repository=flexmock(), statistics=flexmock(), list_details=flexmock() + ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_prune( config_filename='test.yaml', repository={'path': 'repo'}, config={}, - hook_context={}, local_borg_version=None, prune_arguments=prune_arguments, global_arguments=global_arguments, @@ -54,14 +53,15 @@ def test_run_prune_bails_if_repository_does_not_match(): 'repositories_match' ).once().and_return(False) flexmock(module.borgmatic.borg.prune).should_receive('prune_archives').never() - prune_arguments = flexmock(repository=flexmock(), stats=flexmock(), list_archives=flexmock()) + prune_arguments = flexmock( + repository=flexmock(), statistics=flexmock(), list_details=flexmock() + ) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_prune( config_filename='test.yaml', repository='repo', config={}, - hook_context={}, local_borg_version=None, prune_arguments=prune_arguments, global_arguments=global_arguments, diff --git a/tests/unit/actions/test_recreate.py b/tests/unit/actions/test_recreate.py new file mode 100644 index 00000000..4250af2c --- /dev/null +++ b/tests/unit/actions/test_recreate.py @@ -0,0 +1,39 @@ +from flexmock import flexmock + +from borgmatic.actions import recreate as module + + +def test_run_recreate_does_not_raise(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.recreate).should_receive('recreate_archive') + + recreate_arguments = flexmock(repository=flexmock(), archive=None) + + module.run_recreate( + repository={'path': 'repo'}, + config={}, + local_borg_version=None, + recreate_arguments=recreate_arguments, + global_arguments=flexmock(), + local_path=None, + remote_path=None, + ) + + +def test_run_recreate_with_archive_does_not_raise(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.recreate).should_receive('recreate_archive') + + recreate_arguments = flexmock(repository=flexmock(), archive='test-archive') + + module.run_recreate( + repository={'path': 'repo'}, + config={}, + local_borg_version=None, + recreate_arguments=recreate_arguments, + global_arguments=flexmock(), + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/actions/test_repo_create.py b/tests/unit/actions/test_repo_create.py index 8bb350b1..0b54818e 100644 --- a/tests/unit/actions/test_repo_create.py +++ b/tests/unit/actions/test_repo_create.py @@ -1,9 +1,10 @@ +import pytest from flexmock import flexmock from borgmatic.actions import repo_create as module -def test_run_repo_create_does_not_raise(): +def test_run_repo_create_with_encryption_mode_argument_does_not_raise(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) flexmock(module.borgmatic.borg.repo_create).should_receive('create_repository') @@ -14,7 +15,7 @@ def test_run_repo_create_does_not_raise(): copy_crypt_key=flexmock(), append_only=flexmock(), storage_quota=flexmock(), - make_parent_dirs=flexmock(), + make_parent_directories=flexmock(), ) module.run_repo_create( @@ -28,6 +29,57 @@ def test_run_repo_create_does_not_raise(): ) +def test_run_repo_create_with_encryption_mode_option_does_not_raise(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.repo_create).should_receive('create_repository') + arguments = flexmock( + encryption_mode=None, + source_repository=flexmock(), + repository=flexmock(), + copy_crypt_key=flexmock(), + append_only=flexmock(), + storage_quota=flexmock(), + make_parent_directories=flexmock(), + ) + + module.run_repo_create( + repository={'path': 'repo', 'encryption': flexmock()}, + config={}, + local_borg_version=None, + repo_create_arguments=arguments, + global_arguments=flexmock(dry_run=False), + local_path=None, + remote_path=None, + ) + + +def test_run_repo_create_without_encryption_mode_raises(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.repo_create).should_receive('create_repository') + arguments = flexmock( + encryption_mode=None, + source_repository=flexmock(), + repository=flexmock(), + copy_crypt_key=flexmock(), + append_only=flexmock(), + storage_quota=flexmock(), + make_parent_directories=flexmock(), + ) + + with pytest.raises(ValueError): + module.run_repo_create( + repository={'path': 'repo'}, + config={}, + local_borg_version=None, + repo_create_arguments=arguments, + global_arguments=flexmock(dry_run=False), + local_path=None, + remote_path=None, + ) + + def test_run_repo_create_bails_if_repository_does_not_match(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return( @@ -41,7 +93,7 @@ def test_run_repo_create_bails_if_repository_does_not_match(): copy_crypt_key=flexmock(), append_only=flexmock(), storage_quota=flexmock(), - make_parent_dirs=flexmock(), + make_parent_directories=flexmock(), ) module.run_repo_create( @@ -53,3 +105,91 @@ def test_run_repo_create_bails_if_repository_does_not_match(): local_path=None, remote_path=None, ) + + +def test_run_repo_create_favors_flags_over_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.repo_create).should_receive('create_repository').with_args( + object, + object, + object, + object, + object, + object, + object, + object, + append_only=False, + storage_quota=0, + make_parent_directories=False, + local_path=object, + remote_path=object, + ).once() + arguments = flexmock( + encryption_mode=flexmock(), + source_repository=flexmock(), + repository=flexmock(), + copy_crypt_key=flexmock(), + append_only=False, + storage_quota=0, + make_parent_directories=False, + ) + + module.run_repo_create( + repository={ + 'path': 'repo', + 'append_only': True, + 'storage_quota': '10G', + 'make_parent_directories': True, + }, + config={}, + local_borg_version=None, + repo_create_arguments=arguments, + global_arguments=flexmock(dry_run=False), + local_path=None, + remote_path=None, + ) + + +def test_run_repo_create_defaults_to_config(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.config.validate).should_receive('repositories_match').and_return(True) + flexmock(module.borgmatic.borg.repo_create).should_receive('create_repository').with_args( + object, + object, + object, + object, + object, + object, + object, + object, + append_only=True, + storage_quota='10G', + make_parent_directories=True, + local_path=object, + remote_path=object, + ).once() + arguments = flexmock( + encryption_mode=flexmock(), + source_repository=flexmock(), + repository=flexmock(), + copy_crypt_key=flexmock(), + append_only=None, + storage_quota=None, + make_parent_directories=None, + ) + + module.run_repo_create( + repository={ + 'path': 'repo', + 'append_only': True, + 'storage_quota': '10G', + 'make_parent_directories': True, + }, + config={}, + local_borg_version=None, + repo_create_arguments=arguments, + global_arguments=flexmock(dry_run=False), + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/actions/test_transfer.py b/tests/unit/actions/test_transfer.py index 03d259be..be4eda25 100644 --- a/tests/unit/actions/test_transfer.py +++ b/tests/unit/actions/test_transfer.py @@ -1,3 +1,4 @@ +import pytest from flexmock import flexmock from borgmatic.actions import transfer as module @@ -6,7 +7,7 @@ from borgmatic.actions import transfer as module def test_run_transfer_does_not_raise(): flexmock(module.logger).answer = lambda message: None flexmock(module.borgmatic.borg.transfer).should_receive('transfer_archives') - transfer_arguments = flexmock() + transfer_arguments = flexmock(archive=None) global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) module.run_transfer( @@ -18,3 +19,21 @@ def test_run_transfer_does_not_raise(): local_path=None, remote_path=None, ) + + +def test_run_transfer_with_archive_and_match_archives_raises(): + flexmock(module.logger).answer = lambda message: None + flexmock(module.borgmatic.borg.transfer).should_receive('transfer_archives') + transfer_arguments = flexmock(archive='foo') + global_arguments = flexmock(monitoring_verbosity=1, dry_run=False) + + with pytest.raises(ValueError): + module.run_transfer( + repository={'path': 'repo'}, + config={'match_archives': 'foo*'}, + local_borg_version=None, + transfer_arguments=transfer_arguments, + global_arguments=global_arguments, + local_path=None, + remote_path=None, + ) diff --git a/tests/unit/borg/test_borg.py b/tests/unit/borg/test_borg.py index 573ee21e..5f346917 100644 --- a/tests/unit/borg/test_borg.py +++ b/tests/unit/borg/test_borg.py @@ -17,7 +17,7 @@ def test_run_arbitrary_borg_calls_borg_with_flags(): ('borg', 'break-lock', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -41,7 +41,7 @@ def test_run_arbitrary_borg_with_log_info_calls_borg_with_info_flag(): ('borg', 'break-lock', '--info', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -66,7 +66,7 @@ def test_run_arbitrary_borg_with_log_debug_calls_borg_with_debug_flag(): ('borg', 'break-lock', '--debug', '--show-rc', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -94,7 +94,7 @@ def test_run_arbitrary_borg_with_lock_wait_calls_borg_with_lock_wait_flags(): ('borg', 'break-lock', '--lock-wait', '5', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -118,7 +118,7 @@ def test_run_arbitrary_borg_with_archive_calls_borg_with_archive_flag(): ('borg', 'break-lock', "'::$ARCHIVE'"), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': 'archive'}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': 'archive'}, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -143,7 +143,7 @@ def test_run_arbitrary_borg_with_local_path_calls_borg_via_local_path(): ('borg1', 'break-lock', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg1', borg_exit_codes=None, @@ -169,7 +169,7 @@ def test_run_arbitrary_borg_with_exit_codes_calls_borg_using_them(): ('borg', 'break-lock', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg', borg_exit_codes=borg_exit_codes, @@ -195,7 +195,7 @@ def test_run_arbitrary_borg_with_remote_path_calls_borg_with_remote_path_flags() ('borg', 'break-lock', '--remote-path', 'borg1', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -222,7 +222,7 @@ def test_run_arbitrary_borg_with_remote_path_injection_attack_gets_escaped(): ('borg', 'break-lock', '--remote-path', "'borg1; naughty-command'", '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -247,7 +247,7 @@ def test_run_arbitrary_borg_passes_borg_specific_flags_to_borg(): ('borg', 'list', '--progress', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -271,7 +271,7 @@ def test_run_arbitrary_borg_omits_dash_dash_in_flags_passed_to_borg(): ('borg', 'break-lock', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -295,7 +295,7 @@ def test_run_arbitrary_borg_without_borg_specific_flags_does_not_raise(): ('borg',), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -319,7 +319,7 @@ def test_run_arbitrary_borg_passes_key_sub_command_to_borg_before_injected_flags ('borg', 'key', 'export', '--info', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -344,7 +344,7 @@ def test_run_arbitrary_borg_passes_debug_sub_command_to_borg_before_injected_fla ('borg', 'debug', 'dump-manifest', '--info', '::', 'path'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -371,7 +371,7 @@ def test_run_arbitrary_borg_calls_borg_with_working_directory(): ('borg', 'break-lock', '::'), output_file=module.borgmatic.execute.DO_NOT_CAPTURE, shell=True, - extra_environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, + environment={'BORG_REPO': 'repo', 'ARCHIVE': ''}, working_directory='/working/dir', borg_local_path='borg', borg_exit_codes=None, diff --git a/tests/unit/borg/test_break_lock.py b/tests/unit/borg/test_break_lock.py index 202cd7d6..8368a19a 100644 --- a/tests/unit/borg/test_break_lock.py +++ b/tests/unit/borg/test_break_lock.py @@ -14,7 +14,7 @@ def insert_execute_command_mock(command, working_directory=None, borg_exit_codes ) flexmock(module).should_receive('execute_command').with_args( command, - extra_environment=None, + environment=None, working_directory=working_directory, borg_local_path=command[0], borg_exit_codes=borg_exit_codes, diff --git a/tests/unit/borg/test_change_passphrase.py b/tests/unit/borg/test_change_passphrase.py index c7b75c7c..ef4b7b35 100644 --- a/tests/unit/borg/test_change_passphrase.py +++ b/tests/unit/borg/test_change_passphrase.py @@ -25,7 +25,7 @@ def insert_execute_command_mock( command, output_file=output_file, output_log_level=module.logging.ANSWER, - extra_environment=None, + environment=None, working_directory=working_directory, borg_local_path=command[0], borg_exit_codes=borg_exit_codes, diff --git a/tests/unit/borg/test_check.py b/tests/unit/borg/test_check.py index 3fadc2fd..0a1f5361 100644 --- a/tests/unit/borg/test_check.py +++ b/tests/unit/borg/test_check.py @@ -18,7 +18,7 @@ def insert_execute_command_mock( flexmock(module).should_receive('execute_command').with_args( command, output_file=output_file, - extra_environment=None, + environment=None, working_directory=working_directory, borg_local_path=command[0], borg_exit_codes=borg_exit_codes, @@ -155,22 +155,6 @@ def test_make_archive_filter_flags_with_data_check_and_prefix_includes_match_arc assert flags == ('--match-archives', 'sh:foo-*') -def test_make_archive_filter_flags_prefers_check_arguments_match_archives_to_config_match_archives(): - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - 'baz-*', None, '1.2.3' - ).and_return(('--match-archives', 'sh:baz-*')) - - flags = module.make_archive_filter_flags( - '1.2.3', - {'match_archives': 'bar-{now}', 'prefix': ''}, # noqa: FS003 - ('archives',), - check_arguments=flexmock(match_archives='baz-*'), - ) - - assert flags == ('--match-archives', 'sh:baz-*') - - def test_make_archive_filter_flags_with_archives_check_and_empty_prefix_uses_archive_name_format_instead(): flexmock(module.feature).should_receive('available').and_return(True) flexmock(module.flags).should_receive('make_match_archives_flags').with_args( @@ -332,7 +316,7 @@ def test_get_repository_id_with_missing_json_keys_raises(): def test_check_archives_with_progress_passes_through_to_borg(): - config = {} + config = {'progress': True} flexmock(module).should_receive('make_check_name_flags').with_args( {'repository'}, () ).and_return(()) @@ -342,7 +326,7 @@ def test_check_archives_with_progress_passes_through_to_borg(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'check', '--progress', 'repo'), output_file=module.DO_NOT_CAPTURE, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -353,7 +337,7 @@ def test_check_archives_with_progress_passes_through_to_borg(): config=config, local_borg_version='1.2.3', check_arguments=flexmock( - progress=True, + progress=None, repair=None, only_checks=None, force=None, @@ -377,7 +361,7 @@ def test_check_archives_with_repair_passes_through_to_borg(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'check', '--repair', 'repo'), output_file=module.DO_NOT_CAPTURE, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -412,7 +396,7 @@ def test_check_archives_with_max_duration_flag_passes_through_to_borg(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'check', '--max-duration', '33', 'repo'), output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -447,7 +431,7 @@ def test_check_archives_with_max_duration_option_passes_through_to_borg(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'check', '--max-duration', '33', 'repo'), output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -610,7 +594,7 @@ def test_check_archives_with_max_duration_flag_overrides_max_duration_option(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'check', '--max-duration', '44', 'repo'), output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -963,7 +947,7 @@ def test_check_archives_with_match_archives_passes_through_to_borg(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'check', '--match-archives', 'foo-*', 'repo'), output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, diff --git a/tests/unit/borg/test_compact.py b/tests/unit/borg/test_compact.py index 18df3d13..7fff7728 100644 --- a/tests/unit/borg/test_compact.py +++ b/tests/unit/borg/test_compact.py @@ -17,7 +17,7 @@ def insert_execute_command_mock( flexmock(module).should_receive('execute_command').with_args( compact_command, output_log_level=output_log_level, - extra_environment=None, + environment=None, working_directory=working_directory, borg_local_path=compact_command[0], borg_exit_codes=borg_exit_codes, @@ -27,7 +27,7 @@ def insert_execute_command_mock( COMPACT_COMMAND = ('borg', 'compact') -def test_compact_segments_calls_borg_with_parameters(): +def test_compact_segments_calls_borg_with_flags(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('repo',), logging.INFO) @@ -40,7 +40,7 @@ def test_compact_segments_calls_borg_with_parameters(): ) -def test_compact_segments_with_log_info_calls_borg_with_info_parameter(): +def test_compact_segments_with_log_info_calls_borg_with_info_flag(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--info', 'repo'), logging.INFO) insert_logging_mock(logging.INFO) @@ -54,7 +54,7 @@ def test_compact_segments_with_log_info_calls_borg_with_info_parameter(): ) -def test_compact_segments_with_log_debug_calls_borg_with_debug_parameter(): +def test_compact_segments_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--debug', '--show-rc', 'repo'), logging.INFO) insert_logging_mock(logging.DEBUG) @@ -110,7 +110,7 @@ def test_compact_segments_with_exit_codes_calls_borg_using_them(): ) -def test_compact_segments_with_remote_path_calls_borg_with_remote_path_parameters(): +def test_compact_segments_with_remote_path_calls_borg_with_remote_path_flags(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--remote-path', 'borg1', 'repo'), logging.INFO) @@ -124,21 +124,20 @@ def test_compact_segments_with_remote_path_calls_borg_with_remote_path_parameter ) -def test_compact_segments_with_progress_calls_borg_with_progress_parameter(): +def test_compact_segments_with_progress_calls_borg_with_progress_flag(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--progress', 'repo'), logging.INFO) module.compact_segments( dry_run=False, repository_path='repo', - config={}, + config={'progress': True}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - progress=True, ) -def test_compact_segments_with_cleanup_commits_calls_borg_with_cleanup_commits_parameter(): +def test_compact_segments_with_cleanup_commits_calls_borg_with_cleanup_commits_flag(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--cleanup-commits', 'repo'), logging.INFO) @@ -152,21 +151,20 @@ def test_compact_segments_with_cleanup_commits_calls_borg_with_cleanup_commits_p ) -def test_compact_segments_with_threshold_calls_borg_with_threshold_parameter(): +def test_compact_segments_with_threshold_calls_borg_with_threshold_flag(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--threshold', '20', 'repo'), logging.INFO) module.compact_segments( dry_run=False, repository_path='repo', - config={}, + config={'compact_threshold': 20}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - threshold=20, ) -def test_compact_segments_with_umask_calls_borg_with_umask_parameters(): +def test_compact_segments_with_umask_calls_borg_with_umask_flags(): config = {'umask': '077'} flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--umask', '077', 'repo'), logging.INFO) @@ -180,7 +178,7 @@ def test_compact_segments_with_umask_calls_borg_with_umask_parameters(): ) -def test_compact_segments_with_log_json_calls_borg_with_log_json_parameters(): +def test_compact_segments_with_log_json_calls_borg_with_log_json_flags(): flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--log-json', 'repo'), logging.INFO) @@ -193,7 +191,7 @@ def test_compact_segments_with_log_json_calls_borg_with_log_json_parameters(): ) -def test_compact_segments_with_lock_wait_calls_borg_with_lock_wait_parameters(): +def test_compact_segments_with_lock_wait_calls_borg_with_lock_wait_flags(): config = {'lock_wait': 5} flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) insert_execute_command_mock(COMPACT_COMMAND + ('--lock-wait', '5', 'repo'), logging.INFO) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 6015004a..80b80ee1 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -4,7 +4,7 @@ import pytest from flexmock import flexmock from borgmatic.borg import create as module -from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type +from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type from ..test_verbosity import insert_logging_mock @@ -185,6 +185,12 @@ def test_any_parent_directories_treats_unrelated_paths_as_non_match(): def test_collect_special_file_paths_parses_special_files_from_borg_dry_run_file_list(): + flexmock(module.flags).should_receive('omit_flag').replace_with( + lambda arguments, flag: arguments + ) + flexmock(module.flags).should_receive('omit_flag_and_value').replace_with( + lambda arguments, flag: arguments + ) flexmock(module.environment).should_receive('make_environment').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').and_return( 'Processing files ...\n- /foo\n+ /bar\n- /baz' @@ -204,6 +210,12 @@ def test_collect_special_file_paths_parses_special_files_from_borg_dry_run_file_ def test_collect_special_file_paths_skips_borgmatic_runtime_directory(): + flexmock(module.flags).should_receive('omit_flag').replace_with( + lambda arguments, flag: arguments + ) + flexmock(module.flags).should_receive('omit_flag_and_value').replace_with( + lambda arguments, flag: arguments + ) flexmock(module.environment).should_receive('make_environment').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').and_return( '+ /foo\n- /run/borgmatic/bar\n- /baz' @@ -231,6 +243,12 @@ def test_collect_special_file_paths_skips_borgmatic_runtime_directory(): def test_collect_special_file_paths_with_borgmatic_runtime_directory_missing_from_paths_output_errors(): + flexmock(module.flags).should_receive('omit_flag').replace_with( + lambda arguments, flag: arguments + ) + flexmock(module.flags).should_receive('omit_flag_and_value').replace_with( + lambda arguments, flag: arguments + ) flexmock(module.environment).should_receive('make_environment').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').and_return( '+ /foo\n- /bar\n- /baz' @@ -251,6 +269,12 @@ def test_collect_special_file_paths_with_borgmatic_runtime_directory_missing_fro def test_collect_special_file_paths_with_dry_run_and_borgmatic_runtime_directory_missing_from_paths_output_does_not_raise(): + flexmock(module.flags).should_receive('omit_flag').replace_with( + lambda arguments, flag: arguments + ) + flexmock(module.flags).should_receive('omit_flag_and_value').replace_with( + lambda arguments, flag: arguments + ) flexmock(module.environment).should_receive('make_environment').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').and_return( '+ /foo\n- /bar\n- /baz' @@ -270,6 +294,12 @@ def test_collect_special_file_paths_with_dry_run_and_borgmatic_runtime_directory def test_collect_special_file_paths_excludes_non_special_files(): + flexmock(module.flags).should_receive('omit_flag').replace_with( + lambda arguments, flag: arguments + ) + flexmock(module.flags).should_receive('omit_flag_and_value').replace_with( + lambda arguments, flag: arguments + ) flexmock(module.environment).should_receive('make_environment').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').and_return( '+ /foo\n+ /bar\n+ /baz' @@ -290,30 +320,6 @@ def test_collect_special_file_paths_excludes_non_special_files(): ) == ('/foo', '/baz') -def test_collect_special_file_paths_omits_exclude_no_dump_flag_from_command(): - flexmock(module.environment).should_receive('make_environment').and_return(None) - flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('borg', 'create', '--dry-run', '--list'), - capture_stderr=True, - working_directory=None, - extra_environment=None, - borg_local_path='borg', - borg_exit_codes=None, - ).and_return('Processing files ...\n- /foo\n+ /bar\n- /baz').once() - flexmock(module).should_receive('special_file').and_return(True) - flexmock(module.os.path).should_receive('exists').and_return(False) - flexmock(module).should_receive('any_parent_directories').never() - - module.collect_special_file_paths( - dry_run=False, - create_command=('borg', 'create', '--exclude-nodump'), - config={}, - local_path='borg', - working_directory=None, - borgmatic_runtime_directory='/run/borgmatic', - ) - - DEFAULT_ARCHIVE_NAME = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}' # noqa: FS003 REPO_ARCHIVE = (f'repo::{DEFAULT_ARCHIVE_NAME}',) @@ -625,12 +631,12 @@ def test_make_base_create_command_includes_list_flags_in_borg_command(): config={ 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], + 'list_details': True, }, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/run/borgmatic', - list_files=True, ) assert create_flags == ('borg', 'create', '--list', '--filter', 'FOO') @@ -663,6 +669,7 @@ def test_make_base_create_command_with_stream_processes_ignores_read_special_fal '/dev/null', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, + source=Pattern_source.INTERNAL, ), ), '/run/borgmatic', @@ -713,6 +720,7 @@ def test_make_base_create_command_without_patterns_and_with_stream_processes_ign '/dev/null', Pattern_type.NO_RECURSE, Pattern_style.FNMATCH, + source=Pattern_source.INTERNAL, ), ), '/run/borgmatic', @@ -954,7 +962,7 @@ def test_make_base_create_command_with_non_existent_directory_and_source_directo ) -def test_create_archive_calls_borg_with_parameters(): +def test_create_archive_calls_borg_with_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_base_create_command').and_return( @@ -969,7 +977,7 @@ def test_create_archive_calls_borg_with_parameters(): borg_local_path='borg', borg_exit_codes=None, working_directory=None, - extra_environment=None, + environment=None, ) module.create_archive( @@ -1003,7 +1011,7 @@ def test_create_archive_calls_borg_with_environment(): borg_local_path='borg', borg_exit_codes=None, working_directory=None, - extra_environment=environment, + environment=environment, ) module.create_archive( @@ -1021,7 +1029,7 @@ def test_create_archive_calls_borg_with_environment(): ) -def test_create_archive_with_log_info_calls_borg_with_info_parameter(): +def test_create_archive_with_log_info_calls_borg_with_info_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_base_create_command').and_return( @@ -1036,7 +1044,7 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter(): borg_local_path='borg', borg_exit_codes=None, working_directory=None, - extra_environment=None, + environment=None, ) insert_logging_mock(logging.INFO) @@ -1066,7 +1074,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output(): flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'create', '--json') + REPO_ARCHIVE, working_directory=None, - extra_environment=None, + environment=None, borg_local_path='borg', borg_exit_codes=None, ) @@ -1088,7 +1096,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output(): ) -def test_create_archive_with_log_debug_calls_borg_with_debug_parameter(): +def test_create_archive_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_base_create_command').and_return( @@ -1103,7 +1111,7 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter(): borg_local_path='borg', borg_exit_codes=None, working_directory=None, - extra_environment=None, + environment=None, ) insert_logging_mock(logging.DEBUG) @@ -1133,7 +1141,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output(): flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'create', '--json') + REPO_ARCHIVE, working_directory=None, - extra_environment=None, + environment=None, borg_local_path='borg', borg_exit_codes=None, ) @@ -1172,7 +1180,7 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats(): borg_local_path='borg', borg_exit_codes=None, working_directory=None, - extra_environment=None, + environment=None, ) insert_logging_mock(logging.INFO) @@ -1188,7 +1196,6 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats(): local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', - stats=True, ) @@ -1209,7 +1216,7 @@ def test_create_archive_with_working_directory_calls_borg_with_working_directory borg_local_path='borg', borg_exit_codes=None, working_directory='/working/dir', - extra_environment=None, + environment=None, ) module.create_archive( @@ -1244,7 +1251,7 @@ def test_create_archive_with_exit_codes_calls_borg_using_them(): borg_local_path='borg', borg_exit_codes=borg_exit_codes, working_directory=None, - extra_environment=None, + environment=None, ) module.create_archive( @@ -1263,7 +1270,7 @@ def test_create_archive_with_exit_codes_calls_borg_using_them(): ) -def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_output_log_level(): +def test_create_archive_with_stats_calls_borg_with_stats_flag_and_answer_output_log_level(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_base_create_command').and_return( @@ -1278,7 +1285,7 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_ou borg_local_path='borg', borg_exit_codes=None, working_directory=None, - extra_environment=None, + environment=None, ) module.create_archive( @@ -1288,12 +1295,12 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_answer_ou 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'statistics': True, }, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', - stats=True, ) @@ -1316,7 +1323,7 @@ def test_create_archive_with_files_calls_borg_with_answer_output_log_level(): borg_local_path='borg', borg_exit_codes=None, working_directory=None, - extra_environment=None, + environment=None, ) module.create_archive( @@ -1326,16 +1333,16 @@ def test_create_archive_with_files_calls_borg_with_answer_output_log_level(): 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'list_details': True, }, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', - list_files=True, ) -def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_parameter_and_no_list(): +def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_flag_and_no_list(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_base_create_command').and_return( @@ -1350,7 +1357,7 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para borg_local_path='borg', borg_exit_codes=None, working_directory=None, - extra_environment=None, + environment=None, ) insert_logging_mock(logging.INFO) @@ -1361,16 +1368,16 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'progress': True, }, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', - progress=True, ) -def test_create_archive_with_progress_calls_borg_with_progress_parameter(): +def test_create_archive_with_progress_calls_borg_with_progress_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_base_create_command').and_return( @@ -1385,7 +1392,7 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter(): borg_local_path='borg', borg_exit_codes=None, working_directory=None, - extra_environment=None, + environment=None, ) module.create_archive( @@ -1395,16 +1402,16 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter(): 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'progress': True, }, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', - progress=True, ) -def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progress_parameter(): +def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progress_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER processes = flexmock() @@ -1431,7 +1438,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr borg_local_path='borg', borg_exit_codes=None, working_directory=None, - extra_environment=None, + environment=None, ) flexmock(module).should_receive('execute_command_with_processes').with_args( create_command, @@ -1441,7 +1448,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr borg_local_path='borg', borg_exit_codes=None, working_directory=None, - extra_environment=None, + environment=None, ) module.create_archive( @@ -1451,12 +1458,12 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr 'source_directories': ['foo', 'bar'], 'repositories': ['repo'], 'exclude_patterns': None, + 'progress': True, }, patterns=[Pattern('foo'), Pattern('bar')], local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', - progress=True, stream_processes=processes, ) @@ -1472,7 +1479,7 @@ def test_create_archive_with_json_calls_borg_with_json_flag(): flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'create', '--json') + REPO_ARCHIVE, working_directory=None, - extra_environment=None, + environment=None, borg_local_path='borg', borg_exit_codes=None, ).and_return('[]') @@ -1506,7 +1513,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_flag(): flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'create', '--json') + REPO_ARCHIVE, working_directory=None, - extra_environment=None, + environment=None, borg_local_path='borg', borg_exit_codes=None, ).and_return('[]') @@ -1524,7 +1531,6 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_flag(): global_arguments=flexmock(log_json=False), borgmatic_runtime_directory='/borgmatic/run', json=True, - stats=True, ) assert json_output == '[]' @@ -1547,7 +1553,7 @@ def test_create_archive_calls_borg_with_working_directory(): borg_local_path='borg', borg_exit_codes=None, working_directory='/working/dir', - extra_environment=None, + environment=None, ) module.create_archive( diff --git a/tests/unit/borg/test_delete.py b/tests/unit/borg/test_delete.py index c3bf9db7..770f411d 100644 --- a/tests/unit/borg/test_delete.py +++ b/tests/unit/borg/test_delete.py @@ -21,7 +21,7 @@ def test_make_delete_command_includes_log_info(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -43,7 +43,7 @@ def test_make_delete_command_includes_log_debug(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -67,7 +67,7 @@ def test_make_delete_command_includes_dry_run(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=True, log_json=False), local_path='borg', remote_path=None, @@ -91,7 +91,7 @@ def test_make_delete_command_includes_remote_path(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path='borg1', @@ -114,7 +114,7 @@ def test_make_delete_command_includes_umask(): repository={'path': 'repo'}, config={'umask': '077'}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -138,7 +138,7 @@ def test_make_delete_command_includes_log_json(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=True), local_path='borg', remote_path=None, @@ -162,7 +162,7 @@ def test_make_delete_command_includes_lock_wait(): repository={'path': 'repo'}, config={'lock_wait': 5}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -171,7 +171,7 @@ def test_make_delete_command_includes_lock_wait(): assert command == ('borg', 'delete', '--lock-wait', '5', 'repo') -def test_make_delete_command_includes_list(): +def test_make_delete_command_with_list_config_calls_borg_with_list_flag(): flexmock(module.borgmatic.borg.flags).should_receive('make_flags').and_return(()) flexmock(module.borgmatic.borg.flags).should_receive('make_flags').with_args( 'list', True @@ -184,9 +184,9 @@ def test_make_delete_command_includes_list(): command = module.make_delete_command( repository={'path': 'repo'}, - config={}, + config={'list_details': True}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=True, force=0, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=None, force=0, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -207,7 +207,7 @@ def test_make_delete_command_includes_force(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=1, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=1, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -228,7 +228,7 @@ def test_make_delete_command_includes_force_twice(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - delete_arguments=flexmock(list_archives=False, force=2, match_archives=None, archive=None), + delete_arguments=flexmock(list_details=False, force=2, match_archives=None, archive=None), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -252,7 +252,7 @@ def test_make_delete_command_includes_archive(): config={}, local_borg_version='1.2.3', delete_arguments=flexmock( - list_archives=False, force=0, match_archives=None, archive='archive' + list_details=False, force=0, match_archives=None, archive='archive' ), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', @@ -277,7 +277,7 @@ def test_make_delete_command_includes_match_archives(): config={}, local_borg_version='1.2.3', delete_arguments=flexmock( - list_archives=False, force=0, match_archives='sh:foo*', archive='archive' + list_details=False, force=0, match_archives='sh:foo*', archive='archive' ), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', @@ -287,8 +287,12 @@ def test_make_delete_command_includes_match_archives(): assert command == ('borg', 'delete', '--match-archives', 'sh:foo*', 'repo') +LOGGING_ANSWER = flexmock() + + def test_delete_archives_with_archive_calls_borg_delete(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module.borgmatic.borg.repo_delete).should_receive('delete_repository').never() flexmock(module).should_receive('make_delete_command').and_return(flexmock()) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -308,6 +312,7 @@ def test_delete_archives_with_archive_calls_borg_delete(): def test_delete_archives_with_match_archives_calls_borg_delete(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module.borgmatic.borg.repo_delete).should_receive('delete_repository').never() flexmock(module).should_receive('make_delete_command').and_return(flexmock()) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -328,6 +333,7 @@ def test_delete_archives_with_match_archives_calls_borg_delete(): @pytest.mark.parametrize('argument_name', module.ARCHIVE_RELATED_ARGUMENT_NAMES[2:]) def test_delete_archives_with_archive_related_argument_calls_borg_delete(argument_name): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module.borgmatic.borg.repo_delete).should_receive('delete_repository').never() flexmock(module).should_receive('make_delete_command').and_return(flexmock()) flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( @@ -347,6 +353,7 @@ def test_delete_archives_with_archive_related_argument_calls_borg_delete(argumen def test_delete_archives_without_archive_related_argument_calls_borg_repo_delete(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) flexmock(module.borgmatic.borg.repo_delete).should_receive('delete_repository').once() flexmock(module).should_receive('make_delete_command').never() @@ -359,7 +366,7 @@ def test_delete_archives_without_archive_related_argument_calls_borg_repo_delete config={}, local_borg_version=flexmock(), delete_arguments=flexmock( - list_archives=True, force=False, cache_only=False, keep_security_info=False + list_details=True, force=False, cache_only=False, keep_security_info=False ), global_arguments=flexmock(), ) @@ -367,12 +374,13 @@ def test_delete_archives_without_archive_related_argument_calls_borg_repo_delete def test_delete_archives_calls_borg_delete_with_working_directory(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module.borgmatic.borg.repo_delete).should_receive('delete_repository').never() command = flexmock() flexmock(module).should_receive('make_delete_command').and_return(command) - extra_environment = flexmock() + environment = flexmock() flexmock(module.borgmatic.borg.environment).should_receive('make_environment').and_return( - extra_environment + environment ) flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return( '/working/dir' @@ -380,7 +388,7 @@ def test_delete_archives_calls_borg_delete_with_working_directory(): flexmock(module.borgmatic.execute).should_receive('execute_command').with_args( command, output_log_level=logging.ANSWER, - extra_environment=extra_environment, + environment=environment, working_directory='/working/dir', borg_local_path='borg', borg_exit_codes=None, diff --git a/tests/unit/borg/test_environment.py b/tests/unit/borg/test_environment.py index e007664e..46bb33b7 100644 --- a/tests/unit/borg/test_environment.py +++ b/tests/unit/borg/test_environment.py @@ -4,6 +4,12 @@ from borgmatic.borg import environment as module def test_make_environment_with_passcommand_should_call_it_and_set_passphrase_file_descriptor_in_environment(): + flexmock(module.os).should_receive('environ').and_return( + {'USER': 'root', 'BORG_PASSCOMMAND': 'nope'} + ) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).and_return(None) flexmock(module.borgmatic.borg.passcommand).should_receive( 'get_passphrase_from_passcommand' ).and_return('passphrase') @@ -14,113 +20,153 @@ def test_make_environment_with_passcommand_should_call_it_and_set_passphrase_fil environment = module.make_environment({'encryption_passcommand': 'command'}) - assert not environment.get('BORG_PASSCOMMAND') + assert environment.get('BORG_PASSPHRASE') is None + assert environment.get('BORG_PASSCOMMAND') is None assert environment.get('BORG_PASSPHRASE_FD') == '3' -def test_make_environment_with_passphrase_should_set_environment(): +def test_make_environment_with_passphrase_should_set_passphrase_file_descriptor_in_environment(): + flexmock(module.os).should_receive('environ').and_return( + {'USER': 'root', 'BORG_PASSPHRASE': 'nope', 'BORG_PASSCOMMAND': 'nope'} + ) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.borgmatic.borg.passcommand).should_receive( 'get_passphrase_from_passcommand' ).and_return(None) - flexmock(module.os).should_receive('pipe').never() - flexmock(module.os.environ).should_receive('get').and_return(None) + flexmock(module.os).should_receive('pipe').and_return((3, 4)) + flexmock(module.os).should_receive('write') + flexmock(module.os).should_receive('close') + flexmock(module.os).should_receive('set_inheritable') + environment = module.make_environment({'encryption_passphrase': 'pass'}) - assert environment.get('BORG_PASSPHRASE') == 'pass' + assert environment.get('BORG_PASSPHRASE') is None + assert environment.get('BORG_PASSCOMMAND') is None + assert environment.get('BORG_PASSPHRASE_FD') == '3' + + +def test_make_environment_with_credential_tag_passphrase_should_load_it_and_set_passphrase_file_descriptor_in_environment(): + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + config = {'encryption_passphrase': '{credential systemd pass}'} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential', + ).with_args('{credential systemd pass}', config).and_return('pass') + flexmock(module.borgmatic.borg.passcommand).should_receive( + 'get_passphrase_from_passcommand' + ).never() + flexmock(module.os).should_receive('pipe').and_return((3, 4)) + flexmock(module.os).should_receive('write') + flexmock(module.os).should_receive('close') + flexmock(module.os).should_receive('set_inheritable') + + environment = module.make_environment(config) + + assert environment.get('BORG_PASSPHRASE') is None + assert environment.get('BORG_PASSPHRASE_FD') == '3' def test_make_environment_with_ssh_command_should_set_environment(): - flexmock(module.borgmatic.borg.passcommand).should_receive( - 'get_passphrase_from_passcommand' + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' ).and_return(None) flexmock(module.os).should_receive('pipe').never() - flexmock(module.os.environ).should_receive('get').and_return(None) environment = module.make_environment({'ssh_command': 'ssh -C'}) assert environment.get('BORG_RSH') == 'ssh -C' def test_make_environment_without_configuration_sets_certain_environment_variables(): - flexmock(module.borgmatic.borg.passcommand).should_receive( - 'get_passphrase_from_passcommand' + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' ).and_return(None) flexmock(module.os).should_receive('pipe').never() - flexmock(module.os.environ).should_receive('get').and_return(None) environment = module.make_environment({}) # Default environment variables. assert environment == { + 'USER': 'root', 'BORG_EXIT_CODES': 'modern', 'BORG_RELOCATED_REPO_ACCESS_IS_OK': 'no', 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK': 'no', } -def test_make_environment_without_configuration_does_not_set_certain_environment_variables_if_already_set(): - flexmock(module.borgmatic.borg.passcommand).should_receive( - 'get_passphrase_from_passcommand' +def test_make_environment_without_configuration_passes_through_default_environment_variables_untouched(): + flexmock(module.os).should_receive('environ').and_return( + { + 'USER': 'root', + 'BORG_RELOCATED_REPO_ACCESS_IS_OK': 'yup', + 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK': 'nah', + } + ) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' ).and_return(None) flexmock(module.os).should_receive('pipe').never() - flexmock(module.os.environ).should_receive('get').with_args( - 'BORG_RELOCATED_REPO_ACCESS_IS_OK' - ).and_return('yup') - flexmock(module.os.environ).should_receive('get').with_args( - 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK' - ).and_return('nah') environment = module.make_environment({}) - assert environment == {'BORG_EXIT_CODES': 'modern'} + assert environment == { + 'USER': 'root', + 'BORG_RELOCATED_REPO_ACCESS_IS_OK': 'yup', + 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK': 'nah', + 'BORG_EXIT_CODES': 'modern', + } def test_make_environment_with_relocated_repo_access_true_should_set_environment_yes(): - flexmock(module.borgmatic.borg.passcommand).should_receive( - 'get_passphrase_from_passcommand' + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' ).and_return(None) flexmock(module.os).should_receive('pipe').never() - flexmock(module.os.environ).should_receive('get').and_return(None) environment = module.make_environment({'relocated_repo_access_is_ok': True}) assert environment.get('BORG_RELOCATED_REPO_ACCESS_IS_OK') == 'yes' def test_make_environment_with_relocated_repo_access_false_should_set_environment_no(): - flexmock(module.borgmatic.borg.passcommand).should_receive( - 'get_passphrase_from_passcommand' + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' ).and_return(None) flexmock(module.os).should_receive('pipe').never() - flexmock(module.os.environ).should_receive('get').and_return(None) environment = module.make_environment({'relocated_repo_access_is_ok': False}) assert environment.get('BORG_RELOCATED_REPO_ACCESS_IS_OK') == 'no' def test_make_environment_check_i_know_what_i_am_doing_true_should_set_environment_YES(): - flexmock(module.borgmatic.borg.passcommand).should_receive( - 'get_passphrase_from_passcommand' + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' ).and_return(None) flexmock(module.os).should_receive('pipe').never() - flexmock(module.os.environ).should_receive('get').and_return(None) environment = module.make_environment({'check_i_know_what_i_am_doing': True}) assert environment.get('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING') == 'YES' def test_make_environment_check_i_know_what_i_am_doing_false_should_set_environment_NO(): - flexmock(module.borgmatic.borg.passcommand).should_receive( - 'get_passphrase_from_passcommand' + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' ).and_return(None) flexmock(module.os).should_receive('pipe').never() - flexmock(module.os.environ).should_receive('get').and_return(None) environment = module.make_environment({'check_i_know_what_i_am_doing': False}) assert environment.get('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING') == 'NO' def test_make_environment_with_integer_variable_value(): - flexmock(module.borgmatic.borg.passcommand).should_receive( - 'get_passphrase_from_passcommand' + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' ).and_return(None) flexmock(module.os).should_receive('pipe').never() - flexmock(module.os.environ).should_receive('get').and_return(None) environment = module.make_environment({'borg_files_cache_ttl': 40}) + assert environment.get('BORG_FILES_CACHE_TTL') == '40' diff --git a/tests/unit/borg/test_export_key.py b/tests/unit/borg/test_export_key.py index 2d7836a3..2fac04c2 100644 --- a/tests/unit/borg/test_export_key.py +++ b/tests/unit/borg/test_export_key.py @@ -22,7 +22,7 @@ def insert_execute_command_mock( command, output_file=output_file, output_log_level=module.logging.ANSWER, - extra_environment=None, + environment=None, working_directory=working_directory, borg_local_path=command[0], borg_exit_codes=borg_exit_codes, diff --git a/tests/unit/borg/test_export_tar.py b/tests/unit/borg/test_export_tar.py index 53307127..3fb50129 100644 --- a/tests/unit/borg/test_export_tar.py +++ b/tests/unit/borg/test_export_tar.py @@ -23,7 +23,7 @@ def insert_execute_command_mock( command, output_file=None if capture else module.DO_NOT_CAPTURE, output_log_level=output_log_level, - extra_environment=None, + environment=None, working_directory=working_directory, borg_local_path=borg_local_path, borg_exit_codes=borg_exit_codes, @@ -144,7 +144,7 @@ def test_export_tar_archive_calls_borg_with_umask_flags(): ) -def test_export_tar_archive_calls_borg_with_log_json_parameter(): +def test_export_tar_archive_calls_borg_with_log_json_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( @@ -186,7 +186,7 @@ def test_export_tar_archive_calls_borg_with_lock_wait_flags(): ) -def test_export_tar_archive_with_log_info_calls_borg_with_info_parameter(): +def test_export_tar_archive_with_log_info_calls_borg_with_info_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( @@ -230,7 +230,7 @@ def test_export_tar_archive_with_log_debug_calls_borg_with_debug_flags(): ) -def test_export_tar_archive_calls_borg_with_dry_run_parameter(): +def test_export_tar_archive_calls_borg_with_dry_run_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( @@ -273,7 +273,7 @@ def test_export_tar_archive_calls_borg_with_tar_filter_flags(): ) -def test_export_tar_archive_calls_borg_with_list_parameter(): +def test_export_tar_archive_calls_borg_with_list_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( @@ -290,14 +290,13 @@ def test_export_tar_archive_calls_borg_with_list_parameter(): archive='archive', paths=None, destination_path='test.tar', - config={}, + config={'list_details': True}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - list_files=True, ) -def test_export_tar_archive_calls_borg_with_strip_components_parameter(): +def test_export_tar_archive_calls_borg_with_strip_components_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( @@ -320,7 +319,7 @@ def test_export_tar_archive_calls_borg_with_strip_components_parameter(): ) -def test_export_tar_archive_skips_abspath_for_remote_repository_parameter(): +def test_export_tar_archive_skips_abspath_for_remote_repository_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_repository_archive_flags').and_return( diff --git a/tests/unit/borg/test_extract.py b/tests/unit/borg/test_extract.py index bddb2530..a11155f8 100644 --- a/tests/unit/borg/test_extract.py +++ b/tests/unit/borg/test_extract.py @@ -12,7 +12,7 @@ def insert_execute_command_mock(command, destination_path=None, borg_exit_codes= flexmock(module.environment).should_receive('make_environment') flexmock(module).should_receive('execute_command').with_args( command, - extra_environment=None, + environment=None, working_directory=destination_path, borg_local_path=command[0], borg_exit_codes=borg_exit_codes, @@ -580,14 +580,14 @@ def test_extract_archive_with_strip_components_all_and_no_paths_raises(): ) -def test_extract_archive_calls_borg_with_progress_parameter(): +def test_extract_archive_calls_borg_with_progress_flag(): flexmock(module.os.path).should_receive('abspath').and_return('repo') flexmock(module.environment).should_receive('make_environment') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command').with_args( ('borg', 'extract', '--progress', 'repo::archive'), output_file=module.DO_NOT_CAPTURE, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -606,10 +606,9 @@ def test_extract_archive_calls_borg_with_progress_parameter(): repository='repo', archive='archive', paths=None, - config={}, + config={'progress': True}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - progress=True, ) @@ -622,10 +621,9 @@ def test_extract_archive_with_progress_and_extract_to_stdout_raises(): repository='repo', archive='archive', paths=None, - config={}, + config={'progress': True}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), - progress=True, extract_to_stdout=True, ) @@ -639,7 +637,7 @@ def test_extract_archive_calls_borg_with_stdout_parameter_and_returns_process(): ('borg', 'extract', '--stdout', 'repo::archive'), output_file=module.subprocess.PIPE, run_to_completion=False, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -674,7 +672,7 @@ def test_extract_archive_skips_abspath_for_remote_repository(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command').with_args( ('borg', 'extract', 'server:repo::archive'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -710,7 +708,7 @@ def test_extract_archive_uses_configured_working_directory_in_repo_path_and_dest ) flexmock(module.borgmatic.config.validate).should_receive( 'normalize_repository_path' - ).with_args('/working/dir/repo').and_return('/working/dir/repo').once() + ).with_args('repo', '/working/dir').and_return('/working/dir/repo').once() module.extract_archive( dry_run=False, @@ -733,7 +731,7 @@ def test_extract_archive_uses_configured_working_directory_in_repo_path_when_des ) flexmock(module.borgmatic.config.validate).should_receive( 'normalize_repository_path' - ).with_args('/working/dir/repo').and_return('/working/dir/repo').once() + ).with_args('repo', '/working/dir').and_return('/working/dir/repo').once() module.extract_archive( dry_run=False, diff --git a/tests/unit/borg/test_flags.py b/tests/unit/borg/test_flags.py index 38228e1d..63dbab35 100644 --- a/tests/unit/borg/test_flags.py +++ b/tests/unit/borg/test_flags.py @@ -285,3 +285,45 @@ def test_warn_for_aggressive_archive_flags_with_glob_archives_and_json_missing_a flexmock(module.logger).should_receive('warning').never() module.warn_for_aggressive_archive_flags(('borg', '--glob-archives', 'foo*'), '{}') + + +def test_omit_flag_removes_flag_from_arguments(): + assert module.omit_flag(('borg', 'create', '--flag', '--other'), '--flag') == ( + 'borg', + 'create', + '--other', + ) + + +def test_omit_flag_without_flag_present_passes_through_arguments(): + assert module.omit_flag(('borg', 'create', '--other'), '--flag') == ( + 'borg', + 'create', + '--other', + ) + + +def test_omit_flag_and_value_removes_flag_and_value_from_arguments(): + assert module.omit_flag_and_value( + ('borg', 'create', '--flag', 'value', '--other'), '--flag' + ) == ( + 'borg', + 'create', + '--other', + ) + + +def test_omit_flag_and_value_with_equals_sign_removes_flag_and_value_from_arguments(): + assert module.omit_flag_and_value(('borg', 'create', '--flag=value', '--other'), '--flag') == ( + 'borg', + 'create', + '--other', + ) + + +def test_omit_flag_and_value_without_flag_present_passes_through_arguments(): + assert module.omit_flag_and_value(('borg', 'create', '--other'), '--flag') == ( + 'borg', + 'create', + '--other', + ) diff --git a/tests/unit/borg/test_import_key.py b/tests/unit/borg/test_import_key.py new file mode 100644 index 00000000..dfbcd900 --- /dev/null +++ b/tests/unit/borg/test_import_key.py @@ -0,0 +1,279 @@ +import logging + +import pytest +from flexmock import flexmock + +from borgmatic.borg import import_key as module + +from ..test_verbosity import insert_logging_mock + + +def insert_execute_command_mock( + command, input_file=module.DO_NOT_CAPTURE, working_directory=None, borg_exit_codes=None +): + + flexmock(module.environment).should_receive('make_environment') + flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return( + working_directory, + ) + flexmock(module).should_receive('execute_command').with_args( + command, + input_file=input_file, + output_log_level=module.logging.INFO, + environment=None, + working_directory=working_directory, + borg_local_path=command[0], + borg_exit_codes=borg_exit_codes, + ).once() + + +def test_import_key_calls_borg_with_required_flags(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + insert_execute_command_mock(('borg', 'key', 'import', 'repo')) + + module.import_key( + repository_path='repo', + config={}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path=None), + global_arguments=flexmock(dry_run=False, log_json=False), + ) + + +def test_import_key_calls_borg_with_local_path(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + insert_execute_command_mock(('borg1', 'key', 'import', 'repo')) + + module.import_key( + repository_path='repo', + config={}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path=None), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg1', + ) + + +def test_import_key_calls_borg_using_exit_codes(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + borg_exit_codes = flexmock() + insert_execute_command_mock(('borg', 'key', 'import', 'repo'), borg_exit_codes=borg_exit_codes) + + module.import_key( + repository_path='repo', + config={'borg_exit_codes': borg_exit_codes}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path=None), + global_arguments=flexmock(dry_run=False, log_json=False), + ) + + +def test_import_key_calls_borg_with_remote_path_flags(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + insert_execute_command_mock(('borg', 'key', 'import', '--remote-path', 'borg1', 'repo')) + + module.import_key( + repository_path='repo', + config={}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path=None), + global_arguments=flexmock(dry_run=False, log_json=False), + remote_path='borg1', + ) + + +def test_import_key_calls_borg_with_umask_flags(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + insert_execute_command_mock(('borg', 'key', 'import', '--umask', '0770', 'repo')) + + module.import_key( + repository_path='repo', + config={'umask': '0770'}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path=None), + global_arguments=flexmock(dry_run=False, log_json=False), + ) + + +def test_import_key_calls_borg_with_log_json_flags(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + insert_execute_command_mock(('borg', 'key', 'import', '--log-json', 'repo')) + + module.import_key( + repository_path='repo', + config={}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path=None), + global_arguments=flexmock(dry_run=False, log_json=True), + ) + + +def test_import_key_calls_borg_with_lock_wait_flags(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + insert_execute_command_mock(('borg', 'key', 'import', '--lock-wait', '5', 'repo')) + + module.import_key( + repository_path='repo', + config={'lock_wait': '5'}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path=None), + global_arguments=flexmock(dry_run=False, log_json=False), + ) + + +def test_import_key_with_log_info_calls_borg_with_info_parameter(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + insert_execute_command_mock(('borg', 'key', 'import', '--info', 'repo')) + insert_logging_mock(logging.INFO) + + module.import_key( + repository_path='repo', + config={}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path=None), + global_arguments=flexmock(dry_run=False, log_json=False), + ) + + +def test_import_key_with_log_debug_calls_borg_with_debug_flags(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + insert_execute_command_mock(('borg', 'key', 'import', '--debug', '--show-rc', 'repo')) + insert_logging_mock(logging.DEBUG) + + module.import_key( + repository_path='repo', + config={}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path=None), + global_arguments=flexmock(dry_run=False, log_json=False), + ) + + +def test_import_key_calls_borg_with_paper_flags(): + flexmock(module.flags).should_receive('make_flags').and_return(('--paper',)) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + insert_execute_command_mock(('borg', 'key', 'import', '--paper', 'repo')) + + module.import_key( + repository_path='repo', + config={}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=True, path=None), + global_arguments=flexmock(dry_run=False, log_json=False), + ) + + +def test_import_key_calls_borg_with_path_argument(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').with_args('source').and_return(True) + insert_execute_command_mock(('borg', 'key', 'import', 'repo', 'source'), input_file=None) + + module.import_key( + repository_path='repo', + config={}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path='source'), + global_arguments=flexmock(dry_run=False, log_json=False), + ) + + +def test_import_key_with_non_existent_path_raises(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module).should_receive('execute_command').never() + + with pytest.raises(ValueError): + module.import_key( + repository_path='repo', + config={}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path='source'), + global_arguments=flexmock(dry_run=False, log_json=False), + ) + + +def test_import_key_with_stdin_path_calls_borg_without_path_argument(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + insert_execute_command_mock(('borg', 'key', 'import', 'repo')) + + module.import_key( + repository_path='repo', + config={}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path='-'), + global_arguments=flexmock(dry_run=False, log_json=False), + ) + + +def test_import_key_with_dry_run_skips_borg_call(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + flexmock(module).should_receive('execute_command').never() + + module.import_key( + repository_path='repo', + config={}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path=None), + global_arguments=flexmock(dry_run=True, log_json=False), + ) + + +def test_import_key_calls_borg_with_working_directory(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').never() + insert_execute_command_mock(('borg', 'key', 'import', 'repo'), working_directory='/working/dir') + + module.import_key( + repository_path='repo', + config={'working_directory': '/working/dir'}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path=None), + global_arguments=flexmock(dry_run=False, log_json=False), + ) + + +def test_import_key_calls_borg_with_path_argument_and_working_directory(): + flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.os.path).should_receive('exists').with_args('/working/dir/source').and_return( + True + ).once() + insert_execute_command_mock( + ('borg', 'key', 'import', 'repo', 'source'), + input_file=None, + working_directory='/working/dir', + ) + + module.import_key( + repository_path='repo', + config={'working_directory': '/working/dir'}, + local_borg_version='1.2.3', + import_arguments=flexmock(paper=False, path='source'), + global_arguments=flexmock(dry_run=False, log_json=False), + ) diff --git a/tests/unit/borg/test_info.py b/tests/unit/borg/test_info.py index e66f4911..23f416bb 100644 --- a/tests/unit/borg/test_info.py +++ b/tests/unit/borg/test_info.py @@ -380,7 +380,7 @@ def test_make_info_command_with_match_archives_flag_passes_through_to_command(): command = module.make_info_command( repository_path='repo', - config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + config={'archive_name_format': 'bar-{now}', 'match_archives': 'sh:foo-*'}, # noqa: FS003 local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), info_arguments=flexmock(archive=None, json=False, prefix=None, match_archives='sh:foo-*'), @@ -514,7 +514,7 @@ def test_display_archives_info_calls_borg_with_working_directory(): ) flexmock(module).should_receive('execute_command_and_capture_output').with_args( full_command=object, - extra_environment=object, + environment=object, working_directory='/working/dir', borg_local_path=object, borg_exit_codes=object, @@ -523,7 +523,7 @@ def test_display_archives_info_calls_borg_with_working_directory(): flexmock(module).should_receive('execute_command').with_args( full_command=object, output_log_level=object, - extra_environment=object, + environment=object, working_directory='/working/dir', borg_local_path=object, borg_exit_codes=object, diff --git a/tests/unit/borg/test_list.py b/tests/unit/borg/test_list.py index 26803c70..2268e8e5 100644 --- a/tests/unit/borg/test_list.py +++ b/tests/unit/borg/test_list.py @@ -353,7 +353,7 @@ def test_list_archive_calls_borg_with_flags(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'list', 'repo::archive'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -419,7 +419,7 @@ def test_list_archive_calls_borg_with_local_path(): flexmock(module).should_receive('execute_command').with_args( ('borg2', 'list', 'repo::archive'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg2', borg_exit_codes=None, @@ -469,7 +469,7 @@ def test_list_archive_calls_borg_using_exit_codes(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'list', 'repo::archive'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=borg_exit_codes, @@ -507,7 +507,7 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -520,7 +520,7 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'list', 'repo::archive1') + glob_paths, output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -528,7 +528,7 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'list', 'repo::archive2') + glob_paths, output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -576,7 +576,7 @@ def test_list_archive_calls_borg_with_archive(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'list', 'repo::archive'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -699,7 +699,7 @@ def test_list_archive_with_archive_ignores_archive_filter_flag( flexmock(module).should_receive('execute_command').with_args( ('borg', 'list', 'repo::archive'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -759,7 +759,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-list', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -808,7 +808,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes flexmock(module).should_receive('execute_command').with_args( ('borg', 'list', '--repo', 'repo', 'archive1') + glob_paths, output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -816,7 +816,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes flexmock(module).should_receive('execute_command').with_args( ('borg', 'list', '--repo', 'repo', 'archive2') + glob_paths, output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -875,7 +875,7 @@ def test_list_archive_calls_borg_with_working_directory(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'list', 'repo::archive'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory='/working/dir', borg_local_path='borg', borg_exit_codes=None, diff --git a/tests/unit/borg/test_mount.py b/tests/unit/borg/test_mount.py index 46a92408..ee11f266 100644 --- a/tests/unit/borg/test_mount.py +++ b/tests/unit/borg/test_mount.py @@ -14,7 +14,7 @@ def insert_execute_command_mock(command, working_directory=None, borg_exit_codes ) flexmock(module).should_receive('execute_command').with_args( command, - extra_environment=None, + environment=None, working_directory=working_directory, borg_local_path=command[0], borg_exit_codes=borg_exit_codes, @@ -262,7 +262,7 @@ def test_mount_archive_calls_borg_with_foreground_parameter(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'mount', '--foreground', 'repo::archive', '/mnt'), output_file=module.DO_NOT_CAPTURE, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -337,7 +337,7 @@ def test_mount_archive_with_date_based_matching_calls_borg_with_date_based_flags 'repo', '/mnt', ), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, diff --git a/tests/unit/borg/test_passcommand.py b/tests/unit/borg/test_passcommand.py index 198cfd15..a6caea2e 100644 --- a/tests/unit/borg/test_passcommand.py +++ b/tests/unit/borg/test_passcommand.py @@ -3,33 +3,23 @@ from flexmock import flexmock from borgmatic.borg import passcommand as module -def test_run_passcommand_with_passphrase_configured_bails(): - flexmock(module.borgmatic.execute).should_receive('execute_command_and_capture_output').never() - - assert ( - module.run_passcommand('passcommand', passphrase_configured=True, working_directory=None) - is None - ) - - -def test_run_passcommand_without_passphrase_configured_executes_passcommand(): +def test_run_passcommand_does_not_raise(): + module.run_passcommand.cache_clear() flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' - ).and_return('passphrase').once() + ).and_return('passphrase') - assert ( - module.run_passcommand('passcommand', passphrase_configured=False, working_directory=None) - == 'passphrase' - ) + assert module.run_passcommand('passcommand', working_directory=None) == 'passphrase' def test_get_passphrase_from_passcommand_with_configured_passcommand_runs_it(): + module.run_passcommand.cache_clear() flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return( '/working' ) - flexmock(module).should_receive('run_passcommand').with_args( - 'command', False, '/working' - ).and_return('passphrase').once() + flexmock(module).should_receive('run_passcommand').with_args('command', '/working').and_return( + 'passphrase' + ).once() assert ( module.get_passphrase_from_passcommand( @@ -39,33 +29,17 @@ def test_get_passphrase_from_passcommand_with_configured_passcommand_runs_it(): ) -def test_get_passphrase_from_passcommand_with_configured_passphrase_and_passcommand_detects_passphrase(): - flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return( - '/working' - ) - flexmock(module).should_receive('run_passcommand').with_args( - 'command', True, '/working' - ).and_return(None).once() +def test_get_passphrase_from_passcommand_without_configured_passcommand_bails(): + flexmock(module).should_receive('run_passcommand').never() - assert ( - module.get_passphrase_from_passcommand( - {'encryption_passphrase': 'passphrase', 'encryption_passcommand': 'command'}, - ) - is None - ) + assert module.get_passphrase_from_passcommand({}) is None -def test_get_passphrase_from_passcommand_with_configured_blank_passphrase_and_passcommand_detects_passphrase(): - flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return( - '/working' - ) - flexmock(module).should_receive('run_passcommand').with_args( - 'command', True, '/working' - ).and_return(None).once() +def test_run_passcommand_caches_passcommand_after_first_call(): + module.run_passcommand.cache_clear() + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return('passphrase').once() - assert ( - module.get_passphrase_from_passcommand( - {'encryption_passphrase': '', 'encryption_passcommand': 'command'}, - ) - is None - ) + assert module.run_passcommand('passcommand', working_directory=None) == 'passphrase' + assert module.run_passcommand('passcommand', working_directory=None) == 'passphrase' diff --git a/tests/unit/borg/test_prune.py b/tests/unit/borg/test_prune.py index 4880f391..10e8ebf9 100644 --- a/tests/unit/borg/test_prune.py +++ b/tests/unit/borg/test_prune.py @@ -17,7 +17,7 @@ def insert_execute_command_mock( flexmock(module).should_receive('execute_command').with_args( prune_command, output_log_level=output_log_level, - extra_environment=None, + environment=None, working_directory=working_directory, borg_local_path=prune_command[0], borg_exit_codes=borg_exit_codes, @@ -135,32 +135,6 @@ def test_make_prune_flags_without_prefix_uses_archive_name_format_instead(): assert result == expected -def test_make_prune_flags_without_prefix_uses_match_archives_flag_instead_of_option(): - config = { - 'archive_name_format': 'bar-{now}', # noqa: FS003 - 'match_archives': 'foo*', - 'keep_daily': 1, - 'prefix': None, - } - flexmock(module.feature).should_receive('available').and_return(True) - flexmock(module.flags).should_receive('make_match_archives_flags').with_args( - 'baz*', 'bar-{now}', '1.2.3' # noqa: FS003 - ).and_return(('--match-archives', 'sh:bar-*')).once() - - result = module.make_prune_flags( - config, flexmock(match_archives='baz*'), local_borg_version='1.2.3' - ) - - expected = ( - '--keep-daily', - '1', - '--match-archives', - 'sh:bar-*', # noqa: FS003 - ) - - assert result == expected - - def test_make_prune_flags_without_prefix_uses_match_archives_option(): config = { 'archive_name_format': 'bar-{now}', # noqa: FS003 @@ -210,9 +184,12 @@ def test_prune_archives_calls_borg_with_flags(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('repo',), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -228,10 +205,13 @@ def test_prune_archives_with_log_info_calls_borg_with_info_flag(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--info', 'repo'), logging.INFO) insert_logging_mock(logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( repository_path='repo', config={}, @@ -247,10 +227,13 @@ def test_prune_archives_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--debug', '--show-rc', 'repo'), logging.INFO) insert_logging_mock(logging.DEBUG) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( repository_path='repo', config={}, @@ -266,9 +249,12 @@ def test_prune_archives_with_dry_run_calls_borg_with_dry_run_flag(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--dry-run', 'repo'), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( repository_path='repo', config={}, @@ -284,9 +270,12 @@ def test_prune_archives_with_local_path_calls_borg_via_local_path(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock(('borg1',) + PRUNE_COMMAND[1:] + ('repo',), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -303,6 +292,9 @@ def test_prune_archives_with_exit_codes_calls_borg_using_them(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) borg_exit_codes = flexmock() insert_execute_command_mock( ('borg',) + PRUNE_COMMAND[1:] + ('repo',), @@ -310,7 +302,7 @@ def test_prune_archives_with_exit_codes_calls_borg_using_them(): borg_exit_codes=borg_exit_codes, ) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -326,9 +318,12 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_flags(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--remote-path', 'borg1', 'repo'), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -340,36 +335,42 @@ def test_prune_archives_with_remote_path_calls_borg_with_remote_path_flags(): ) -def test_prune_archives_with_stats_calls_borg_with_stats_flag_and_answer_output_log_level(): +def test_prune_archives_with_stats_config_calls_borg_with_stats_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--stats', 'repo'), module.borgmatic.logger.ANSWER) - prune_arguments = flexmock(stats=True, list_archives=False) + prune_arguments = flexmock(statistics=None, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', - config={}, + config={'statistics': True}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, ) -def test_prune_archives_with_files_calls_borg_with_list_flag_and_answer_output_log_level(): +def test_prune_archives_with_list_config_calls_borg_with_list_flag(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--list', 'repo'), module.borgmatic.logger.ANSWER) - prune_arguments = flexmock(stats=False, list_archives=True) + prune_arguments = flexmock(statistics=False, list_details=None) module.prune_archives( dry_run=False, repository_path='repo', - config={}, + config={'list_details': True}, local_borg_version='1.2.3', global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, @@ -382,9 +383,12 @@ def test_prune_archives_with_umask_calls_borg_with_umask_flags(): config = {'umask': '077'} flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--umask', '077', 'repo'), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -400,9 +404,12 @@ def test_prune_archives_with_log_json_calls_borg_with_log_json_flag(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--log-json', 'repo'), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -419,9 +426,12 @@ def test_prune_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): config = {'lock_wait': 5} flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--lock-wait', '5', 'repo'), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -437,9 +447,12 @@ def test_prune_archives_with_extra_borg_options_calls_borg_with_extra_options(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock(PRUNE_COMMAND + ('--extra', '--options', 'repo'), logging.INFO) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -471,6 +484,9 @@ def test_prune_archives_with_date_based_matching_calls_borg_with_date_based_flag ) ) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) flexmock(module.environment).should_receive('make_environment') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command').with_args( @@ -497,14 +513,14 @@ def test_prune_archives_with_date_based_matching_calls_borg_with_date_based_flag 'repo', ), output_log_level=logging.INFO, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, ) prune_arguments = flexmock( - stats=False, list_archives=False, newer='1d', newest='1y', older='1m', oldest='1w' + statistics=False, list_details=False, newer='1d', newest='1y', older='1m', oldest='1w' ) module.prune_archives( dry_run=False, @@ -521,11 +537,14 @@ def test_prune_archives_calls_borg_with_working_directory(): flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '1.2.3' + ).and_return(False) insert_execute_command_mock( PRUNE_COMMAND + ('repo',), logging.INFO, working_directory='/working/dir' ) - prune_arguments = flexmock(stats=False, list_archives=False) + prune_arguments = flexmock(statistics=False, list_details=False) module.prune_archives( dry_run=False, repository_path='repo', @@ -534,3 +553,24 @@ def test_prune_archives_calls_borg_with_working_directory(): global_arguments=flexmock(log_json=False), prune_arguments=prune_arguments, ) + + +def test_prune_archives_calls_borg_without_stats_when_feature_is_not_available(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER + flexmock(module).should_receive('make_prune_flags').and_return(BASE_PRUNE_FLAGS) + flexmock(module.flags).should_receive('make_repository_flags').and_return(('repo',)) + flexmock(module.feature).should_receive('available').with_args( + module.feature.Feature.NO_PRUNE_STATS, '2.0.0b10' + ).and_return(True) + insert_execute_command_mock(PRUNE_COMMAND + ('repo',), logging.ANSWER) + + prune_arguments = flexmock(statistics=True, list_details=False) + module.prune_archives( + dry_run=False, + repository_path='repo', + config={'statistics': True}, + local_borg_version='2.0.0b10', + global_arguments=flexmock(log_json=False), + prune_arguments=prune_arguments, + ) diff --git a/tests/unit/borg/test_recreate.py b/tests/unit/borg/test_recreate.py new file mode 100644 index 00000000..d0d527bf --- /dev/null +++ b/tests/unit/borg/test_recreate.py @@ -0,0 +1,812 @@ +import logging +import shlex + +from flexmock import flexmock + +from borgmatic.borg import recreate as module + +from ..test_verbosity import insert_logging_mock + + +def insert_execute_command_mock(command, working_directory=None, borg_exit_codes=None): + flexmock(module.borgmatic.borg.environment).should_receive('make_environment') + flexmock(module.borgmatic.execute).should_receive('execute_command').with_args( + full_command=command, + output_log_level=module.logging.INFO, + environment=None, + working_directory=working_directory, + borg_local_path=command[0], + borg_exit_codes=borg_exit_codes, + ).once() + + +def test_recreate_archive_dry_run_skips_execution(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return( + ( + '--repo', + 'repo', + ) + ) + flexmock(module.borgmatic.execute).should_receive('execute_command').never() + + recreate_arguments = flexmock( + repository=flexmock(), + list=None, + target=None, + comment=None, + timestamp=None, + match_archives=None, + ) + + result = module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=recreate_arguments, + global_arguments=flexmock(log_json=False, dry_run=True), + local_path='borg', + ) + + assert result is None + + +def test_recreate_calls_borg_with_required_flags(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock(('borg', 'recreate', '--repo', 'repo')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives=None, + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + remote_path=None, + patterns=None, + ) + + +def test_recreate_with_remote_path(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock(('borg', 'recreate', '--remote-path', 'borg1', '--repo', 'repo')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives=None, + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + remote_path='borg1', + patterns=None, + ) + + +def test_recreate_with_lock_wait(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock(('borg', 'recreate', '--lock-wait', '5', '--repo', 'repo')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={'lock_wait': '5'}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives=None, + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_log_info(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock(('borg', 'recreate', '--info', '--repo', 'repo')) + + insert_logging_mock(logging.INFO) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives=None, + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_log_debug(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock(('borg', 'recreate', '--debug', '--show-rc', '--repo', 'repo')) + insert_logging_mock(logging.DEBUG) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives=None, + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_log_json(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock(('borg', 'recreate', '--log-json', '--repo', 'repo')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives=None, + ), + global_arguments=flexmock(dry_run=False, log_json=True), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_list_config_calls_borg_with_list_flag(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return( + ( + '--repo', + 'repo', + ) + ) + flexmock(module).should_receive('make_list_filter_flags').and_return('AME+-') + insert_execute_command_mock( + ('borg', 'recreate', '--list', '--filter', 'AME+-', '--repo', 'repo') + ) + + module.recreate_archive( + repository='repo', + archive='archive', + config={'list_details': True}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives=None, + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_patterns_from_flag(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return( + ( + '--repo', + 'repo', + ) + ) + mock_patterns_file = flexmock(name='patterns_file') + flexmock(module).should_receive('write_patterns_file').and_return(mock_patterns_file) + insert_execute_command_mock( + ('borg', 'recreate', '--patterns-from', 'patterns_file', '--repo', 'repo') + ) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives=None, + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=['pattern1', 'pattern2'], + ) + + +def test_recreate_with_exclude_flags(): + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return( + ( + '--repo', + 'repo', + ) + ) + flexmock(module).should_receive('make_exclude_flags').and_return(('--exclude', 'pattern')) + insert_execute_command_mock(('borg', 'recreate', '--exclude', 'pattern', '--repo', 'repo')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={'exclude_patterns': ['pattern']}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives=None, + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_target_flag(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock(('borg', 'recreate', '--target', 'new-archive', '--repo', 'repo')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target='new-archive', + comment=None, + timestamp=None, + match_archives=None, + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_comment_flag(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock( + ('borg', 'recreate', '--comment', shlex.quote('This is a test comment'), '--repo', 'repo') + ) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment='This is a test comment', + timestamp=None, + match_archives=None, + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_timestamp_flag(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock( + ('borg', 'recreate', '--timestamp', '2023-10-01T12:00:00', '--repo', 'repo') + ) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp='2023-10-01T12:00:00', + match_archives=None, + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_compression_flag(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock(('borg', 'recreate', '--compression', 'lz4', '--repo', 'repo')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={'compression': 'lz4'}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives=None, + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_chunker_params_flag(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock( + ('borg', 'recreate', '--chunker-params', '19,23,21,4095', '--repo', 'repo') + ) + + module.recreate_archive( + repository='repo', + archive='archive', + config={'chunker_params': '19,23,21,4095'}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives=None, + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_recompress_flag(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock(('borg', 'recreate', '--recompress', 'always', '--repo', 'repo')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={'recompress': 'always'}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives=None, + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_match_archives_star(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock(('borg', 'recreate', '--repo', 'repo')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives='*', + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_match_archives_regex(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock(('borg', 'recreate', '--repo', 'repo')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives='re:.*', + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_match_archives_shell(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').and_return(()) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return( + ( + '--repo', + 'repo', + ) + ) + insert_execute_command_mock(('borg', 'recreate', '--repo', 'repo')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives='sh:*', + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_match_archives_and_feature_available_calls_borg_with_match_archives(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').with_args( + 'foo-*', None, '1.2.3' + ).and_return(('--match-archives', 'foo-*')) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( + ('--repo', 'repo') + ) + flexmock(module.borgmatic.borg.flags).should_receive('make_repository_archive_flags').never() + insert_execute_command_mock(('borg', 'recreate', '--repo', 'repo', '--match-archives', 'foo-*')) + + module.recreate_archive( + repository='repo', + archive=None, + config={'match_archives': 'foo-*'}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives='foo-*', + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_archives_flag_and_feature_available_calls_borg_with_match_archives(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').with_args( + 'archive', None, '1.2.3' + ).and_return(('--match-archives', 'archive')) + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(True) + flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( + ('--repo', 'repo') + ) + flexmock(module.borgmatic.borg.flags).should_receive('make_repository_archive_flags').never() + insert_execute_command_mock( + ('borg', 'recreate', '--repo', 'repo', '--match-archives', 'archive') + ) + + module.recreate_archive( + repository='repo', + archive='archive', + config={'match_archives': 'foo-*'}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives='foo-*', + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_match_archives_and_feature_not_available_calls_borg_without_match_archives(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').never() + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(False) + flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').and_return( + ('repo',) + ) + flexmock(module.borgmatic.borg.flags).should_receive('make_repository_archive_flags').never() + insert_execute_command_mock(('borg', 'recreate', 'repo')) + + module.recreate_archive( + repository='repo', + archive=None, + config={'match_archives': 'foo-*'}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives='foo-*', + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) + + +def test_recreate_with_archives_flags_and_feature_not_available_calls_borg_with_combined_repo_and_archive(): + flexmock(module.borgmatic.borg.create).should_receive('make_exclude_flags').and_return(()) + flexmock(module.borgmatic.borg.create).should_receive('write_patterns_file').and_return(None) + flexmock(module.borgmatic.borg.create).should_receive('make_list_filter_flags').and_return('') + flexmock(module.borgmatic.borg.flags).should_receive('make_match_archives_flags').never() + flexmock(module.borgmatic.borg.feature).should_receive('available').and_return(False) + flexmock(module.borgmatic.borg.flags).should_receive( + 'make_repository_archive_flags' + ).and_return(('repo::archive',)) + flexmock(module.borgmatic.borg.flags).should_receive('make_repository_flags').never() + insert_execute_command_mock(('borg', 'recreate', 'repo::archive')) + + module.recreate_archive( + repository='repo', + archive='archive', + config={'match_archives': 'foo-*'}, + local_borg_version='1.2.3', + recreate_arguments=flexmock( + list=None, + target=None, + comment=None, + timestamp=None, + match_archives='foo-*', + ), + global_arguments=flexmock(dry_run=False, log_json=False), + local_path='borg', + patterns=None, + ) diff --git a/tests/unit/borg/test_repo_create.py b/tests/unit/borg/test_repo_create.py index 388ebc7b..9e3a674d 100644 --- a/tests/unit/borg/test_repo_create.py +++ b/tests/unit/borg/test_repo_create.py @@ -36,7 +36,7 @@ def insert_repo_create_command_mock( flexmock(module).should_receive('execute_command').with_args( repo_create_command, output_file=module.DO_NOT_CAPTURE, - extra_environment=None, + environment=None, working_directory=working_directory, borg_local_path=repo_create_command[0], borg_exit_codes=borg_exit_codes, @@ -228,7 +228,29 @@ def test_create_repository_with_append_only_calls_borg_with_append_only_flag(): module.create_repository( dry_run=False, repository_path='repo', - config={}, + config={'append_only': True}, + local_borg_version='2.3.4', + global_arguments=flexmock(log_json=False), + encryption_mode='repokey', + append_only=True, + ) + + +def test_create_repository_with_append_only_config_calls_borg_with_append_only_flag(): + insert_repo_info_command_not_found_mock() + insert_repo_create_command_mock(REPO_CREATE_COMMAND + ('--append-only', '--repo', 'repo')) + flexmock(module.feature).should_receive('available').and_return(True) + flexmock(module.flags).should_receive('make_repository_flags').and_return( + ( + '--repo', + 'repo', + ) + ) + + module.create_repository( + dry_run=False, + repository_path='repo', + config={'append_only': True}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', @@ -252,7 +274,7 @@ def test_create_repository_with_storage_quota_calls_borg_with_storage_quota_flag module.create_repository( dry_run=False, repository_path='repo', - config={}, + config={'storage_quota': '5G'}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', @@ -274,11 +296,11 @@ def test_create_repository_with_make_parent_dirs_calls_borg_with_make_parent_dir module.create_repository( dry_run=False, repository_path='repo', - config={}, + config={'make_parent_directories': True}, local_borg_version='2.3.4', global_arguments=flexmock(log_json=False), encryption_mode='repokey', - make_parent_dirs=True, + make_parent_directories=True, ) diff --git a/tests/unit/borg/test_repo_delete.py b/tests/unit/borg/test_repo_delete.py index d490fa5d..4af66085 100644 --- a/tests/unit/borg/test_repo_delete.py +++ b/tests/unit/borg/test_repo_delete.py @@ -19,7 +19,7 @@ def test_make_repo_delete_command_with_feature_available_runs_borg_repo_delete() repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -40,7 +40,7 @@ def test_make_repo_delete_command_without_feature_available_runs_borg_delete(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -62,7 +62,7 @@ def test_make_repo_delete_command_includes_log_info(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -84,7 +84,7 @@ def test_make_repo_delete_command_includes_log_debug(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -108,7 +108,7 @@ def test_make_repo_delete_command_includes_dry_run(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=True, log_json=False), local_path='borg', remote_path=None, @@ -132,7 +132,7 @@ def test_make_repo_delete_command_includes_remote_path(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path='borg1', @@ -155,7 +155,7 @@ def test_make_repo_delete_command_includes_umask(): repository={'path': 'repo'}, config={'umask': '077'}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -179,7 +179,7 @@ def test_make_repo_delete_command_includes_log_json(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=True), local_path='borg', remote_path=None, @@ -203,7 +203,7 @@ def test_make_repo_delete_command_includes_lock_wait(): repository={'path': 'repo'}, config={'lock_wait': 5}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=0), + repo_delete_arguments=flexmock(list_details=False, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -225,9 +225,9 @@ def test_make_repo_delete_command_includes_list(): command = module.make_repo_delete_command( repository={'path': 'repo'}, - config={}, + config={'list_details': True}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=True, force=0), + repo_delete_arguments=flexmock(list_details=True, force=0), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -248,7 +248,7 @@ def test_make_repo_delete_command_includes_force(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=1), + repo_delete_arguments=flexmock(list_details=False, force=1), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -269,7 +269,7 @@ def test_make_repo_delete_command_includes_force_twice(): repository={'path': 'repo'}, config={}, local_borg_version='1.2.3', - repo_delete_arguments=flexmock(list_archives=False, force=2), + repo_delete_arguments=flexmock(list_details=False, force=2), global_arguments=flexmock(dry_run=False, log_json=False), local_path='borg', remote_path=None, @@ -290,7 +290,7 @@ def test_delete_repository_with_defaults_does_not_capture_output(): command, output_log_level=module.logging.ANSWER, output_file=module.borgmatic.execute.DO_NOT_CAPTURE, - extra_environment=object, + environment=object, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -319,7 +319,7 @@ def test_delete_repository_with_force_captures_output(): command, output_log_level=module.logging.ANSWER, output_file=None, - extra_environment=object, + environment=object, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -348,7 +348,7 @@ def test_delete_repository_with_cache_only_captures_output(): command, output_log_level=module.logging.ANSWER, output_file=None, - extra_environment=object, + environment=object, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -379,7 +379,7 @@ def test_delete_repository_calls_borg_with_working_directory(): command, output_log_level=module.logging.ANSWER, output_file=module.borgmatic.execute.DO_NOT_CAPTURE, - extra_environment=object, + environment=object, working_directory='/working/dir', borg_local_path='borg', borg_exit_codes=None, diff --git a/tests/unit/borg/test_repo_info.py b/tests/unit/borg/test_repo_info.py index 6a0cec11..555cb566 100644 --- a/tests/unit/borg/test_repo_info.py +++ b/tests/unit/borg/test_repo_info.py @@ -24,7 +24,7 @@ def test_display_repository_info_calls_borg_with_flags(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -33,7 +33,7 @@ def test_display_repository_info_calls_borg_with_flags(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'repo-info', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -60,7 +60,7 @@ def test_display_repository_info_without_borg_features_calls_borg_with_info_sub_ flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--json', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -69,7 +69,7 @@ def test_display_repository_info_without_borg_features_calls_borg_with_info_sub_ flexmock(module).should_receive('execute_command').with_args( ('borg', 'info', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -101,7 +101,7 @@ def test_display_repository_info_with_log_info_calls_borg_with_info_flag(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--info', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -110,7 +110,7 @@ def test_display_repository_info_with_log_info_calls_borg_with_info_flag(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'repo-info', '--info', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -142,7 +142,7 @@ def test_display_repository_info_with_log_info_and_json_suppresses_most_borg_out flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -178,7 +178,7 @@ def test_display_repository_info_with_log_debug_calls_borg_with_debug_flag(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--debug', '--show-rc', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -187,7 +187,7 @@ def test_display_repository_info_with_log_debug_calls_borg_with_debug_flag(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'repo-info', '--debug', '--show-rc', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -220,7 +220,7 @@ def test_display_repository_info_with_log_debug_and_json_suppresses_most_borg_ou flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -256,7 +256,7 @@ def test_display_repository_info_with_json_calls_borg_with_json_flag(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -291,7 +291,7 @@ def test_display_repository_info_with_local_path_calls_borg_via_local_path(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg1', 'repo-info', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -300,7 +300,7 @@ def test_display_repository_info_with_local_path_calls_borg_via_local_path(): flexmock(module).should_receive('execute_command').with_args( ('borg1', 'repo-info', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg1', borg_exit_codes=None, @@ -334,7 +334,7 @@ def test_display_repository_info_with_exit_codes_calls_borg_using_them(): borg_exit_codes = flexmock() flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=borg_exit_codes, @@ -343,7 +343,7 @@ def test_display_repository_info_with_exit_codes_calls_borg_using_them(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'repo-info', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=borg_exit_codes, @@ -375,7 +375,7 @@ def test_display_repository_info_with_remote_path_calls_borg_with_remote_path_fl flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--remote-path', 'borg1', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -384,7 +384,7 @@ def test_display_repository_info_with_remote_path_calls_borg_with_remote_path_fl flexmock(module).should_receive('execute_command').with_args( ('borg', 'repo-info', '--remote-path', 'borg1', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -417,7 +417,7 @@ def test_display_repository_info_with_umask_calls_borg_with_umask_flags(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--umask', '077', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -426,7 +426,7 @@ def test_display_repository_info_with_umask_calls_borg_with_umask_flags(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'repo-info', '--umask', '077', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -462,7 +462,7 @@ def test_display_repository_info_with_log_json_calls_borg_with_log_json_flags(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--log-json', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -471,7 +471,7 @@ def test_display_repository_info_with_log_json_calls_borg_with_log_json_flags(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'repo-info', '--log-json', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -504,7 +504,7 @@ def test_display_repository_info_with_lock_wait_calls_borg_with_lock_wait_flags( flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--lock-wait', '5', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -513,7 +513,7 @@ def test_display_repository_info_with_lock_wait_calls_borg_with_lock_wait_flags( flexmock(module).should_receive('execute_command').with_args( ('borg', 'repo-info', '--lock-wait', '5', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -547,7 +547,7 @@ def test_display_repository_info_calls_borg_with_working_directory(): ) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'repo-info', '--json', '--repo', 'repo'), - extra_environment=None, + environment=None, working_directory='/working/dir', borg_local_path='borg', borg_exit_codes=None, @@ -556,7 +556,7 @@ def test_display_repository_info_calls_borg_with_working_directory(): flexmock(module).should_receive('execute_command').with_args( ('borg', 'repo-info', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, - extra_environment=None, + environment=None, working_directory='/working/dir', borg_local_path='borg', borg_exit_codes=None, diff --git a/tests/unit/borg/test_repo_list.py b/tests/unit/borg/test_repo_list.py index fec5e37e..73f567f5 100644 --- a/tests/unit/borg/test_repo_list.py +++ b/tests/unit/borg/test_repo_list.py @@ -39,7 +39,7 @@ def test_resolve_archive_name_calls_borg_with_flags(): ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, borg_local_path='borg', borg_exit_codes=None, - extra_environment=None, + environment=None, working_directory=None, ).and_return(expected_archive + '\n') @@ -61,7 +61,7 @@ def test_resolve_archive_name_with_log_info_calls_borg_without_info_flag(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -86,7 +86,7 @@ def test_resolve_archive_name_with_log_debug_calls_borg_without_debug_flag(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -111,7 +111,7 @@ def test_resolve_archive_name_with_local_path_calls_borg_via_local_path(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg1', 'list') + BORG_LIST_LATEST_ARGUMENTS, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg1', borg_exit_codes=None, @@ -137,7 +137,7 @@ def test_resolve_archive_name_with_exit_codes_calls_borg_using_them(): borg_exit_codes = flexmock() flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=borg_exit_codes, @@ -161,7 +161,7 @@ def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_flags flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list', '--remote-path', 'borg1') + BORG_LIST_LATEST_ARGUMENTS, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -186,7 +186,7 @@ def test_resolve_archive_name_with_umask_calls_borg_with_umask_flags(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list', '--umask', '077') + BORG_LIST_LATEST_ARGUMENTS, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -209,7 +209,7 @@ def test_resolve_archive_name_without_archives_raises(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -232,7 +232,7 @@ def test_resolve_archive_name_with_log_json_calls_borg_with_log_json_flags(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list', '--log-json') + BORG_LIST_LATEST_ARGUMENTS, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -257,7 +257,7 @@ def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_flags(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('borg', 'list', '--lock-wait', 'okay') + BORG_LIST_LATEST_ARGUMENTS, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -285,7 +285,7 @@ def test_resolve_archive_name_calls_borg_with_working_directory(): ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, borg_local_path='borg', borg_exit_codes=None, - extra_environment=None, + environment=None, working_directory='/working/dir', ).and_return(expected_archive + '\n') @@ -664,7 +664,7 @@ def test_make_repo_list_command_with_match_archives_calls_borg_with_match_archiv command = module.make_repo_list_command( repository_path='repo', - config={}, + config={'match_archives': 'foo-*'}, local_borg_version='1.2.3', repo_list_arguments=flexmock( archive=None, @@ -773,7 +773,7 @@ def test_list_repository_calls_borg_with_working_directory(): ) flexmock(module).should_receive('execute_command_and_capture_output').with_args( full_command=object, - extra_environment=object, + environment=object, working_directory='/working/dir', borg_local_path=object, borg_exit_codes=object, @@ -782,7 +782,7 @@ def test_list_repository_calls_borg_with_working_directory(): flexmock(module).should_receive('execute_command').with_args( full_command=object, output_log_level=object, - extra_environment=object, + environment=object, working_directory='/working/dir', borg_local_path=object, borg_exit_codes=object, diff --git a/tests/unit/borg/test_transfer.py b/tests/unit/borg/test_transfer.py index 49514861..b2b9886d 100644 --- a/tests/unit/borg/test_transfer.py +++ b/tests/unit/borg/test_transfer.py @@ -21,7 +21,7 @@ def test_transfer_archives_calls_borg_with_flags(): ('borg', 'transfer', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -55,7 +55,7 @@ def test_transfer_archives_with_dry_run_calls_borg_with_dry_run_flag(): ('borg', 'transfer', '--repo', 'repo', '--dry-run'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -86,7 +86,7 @@ def test_transfer_archives_with_log_info_calls_borg_with_info_flag(): ('borg', 'transfer', '--info', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -117,7 +117,7 @@ def test_transfer_archives_with_log_debug_calls_borg_with_debug_flag(): ('borg', 'transfer', '--debug', '--show-rc', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -151,7 +151,7 @@ def test_transfer_archives_with_archive_calls_borg_with_match_archives_flag(): ('borg', 'transfer', '--match-archives', 'archive', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -184,7 +184,7 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl ('borg', 'transfer', '--match-archives', 'sh:foo*', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -193,7 +193,7 @@ def test_transfer_archives_with_match_archives_calls_borg_with_match_archives_fl module.transfer_archives( dry_run=False, repository_path='repo', - config={'archive_name_format': 'bar-{now}'}, # noqa: FS003 + config={'archive_name_format': 'bar-{now}', 'match_archives': 'sh:foo*'}, # noqa: FS003 local_borg_version='2.3.4', transfer_arguments=flexmock( archive=None, progress=None, match_archives='sh:foo*', source_repository=None @@ -217,7 +217,7 @@ def test_transfer_archives_with_archive_name_format_calls_borg_with_match_archiv ('borg', 'transfer', '--match-archives', 'sh:bar-*', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -248,7 +248,7 @@ def test_transfer_archives_with_local_path_calls_borg_via_local_path(): ('borg2', 'transfer', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg2', borg_exit_codes=None, @@ -281,7 +281,7 @@ def test_transfer_archives_with_exit_codes_calls_borg_using_them(): ('borg', 'transfer', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=borg_exit_codes, @@ -315,7 +315,7 @@ def test_transfer_archives_with_remote_path_calls_borg_with_remote_path_flags(): ('borg', 'transfer', '--remote-path', 'borg2', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -349,7 +349,7 @@ def test_transfer_archives_with_umask_calls_borg_with_umask_flags(): ('borg', 'transfer', '--umask', '077', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -383,7 +383,7 @@ def test_transfer_archives_with_log_json_calls_borg_with_log_json_flags(): ('borg', 'transfer', '--log-json', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -418,7 +418,7 @@ def test_transfer_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): ('borg', 'transfer', '--lock-wait', '5', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -436,12 +436,15 @@ def test_transfer_archives_with_lock_wait_calls_borg_with_lock_wait_flags(): ) -def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): +def test_transfer_archives_with_progress_calls_borg_with_progress_flags(): flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.borgmatic.logger.ANSWER flexmock(module.flags).should_receive('make_flags').and_return(()) + flexmock(module.flags).should_receive('make_flags').with_args('progress', True).and_return( + ('--progress',) + ) flexmock(module.flags).should_receive('make_match_archives_flags').and_return(()) - flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--progress',)) + flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(()) flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo')) flexmock(module.environment).should_receive('make_environment') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) @@ -449,7 +452,7 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): ('borg', 'transfer', '--progress', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=module.DO_NOT_CAPTURE, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -458,10 +461,10 @@ def test_transfer_archives_with_progress_calls_borg_with_progress_flag(): module.transfer_archives( dry_run=False, repository_path='repo', - config={}, + config={'progress': True}, local_borg_version='2.3.4', transfer_arguments=flexmock( - archive=None, progress=True, match_archives=None, source_repository=None + archive=None, progress=None, match_archives=None, source_repository=None ), global_arguments=flexmock(log_json=False), ) @@ -484,7 +487,7 @@ def test_transfer_archives_passes_through_arguments_to_borg(argument_name): ('borg', 'transfer', flag_name, 'value', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -521,7 +524,7 @@ def test_transfer_archives_with_source_repository_calls_borg_with_other_repo_fla ('borg', 'transfer', '--repo', 'repo', '--other-repo', 'other'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -566,7 +569,7 @@ def test_transfer_archives_with_date_based_matching_calls_borg_with_date_based_f ), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory=None, borg_local_path='borg', borg_exit_codes=None, @@ -605,7 +608,7 @@ def test_transfer_archives_calls_borg_with_working_directory(): ('borg', 'transfer', '--repo', 'repo'), output_log_level=module.borgmatic.logger.ANSWER, output_file=None, - extra_environment=None, + environment=None, working_directory='/working/dir', borg_local_path='borg', borg_exit_codes=None, diff --git a/tests/unit/borg/test_version.py b/tests/unit/borg/test_version.py index 383eff55..17fc1861 100644 --- a/tests/unit/borg/test_version.py +++ b/tests/unit/borg/test_version.py @@ -23,7 +23,7 @@ def insert_execute_command_and_capture_output_mock( ) flexmock(module).should_receive('execute_command_and_capture_output').with_args( command, - extra_environment=None, + environment=None, working_directory=working_directory, borg_local_path=borg_local_path, borg_exit_codes=borg_exit_codes, diff --git a/tests/unit/commands/completion/test_flag.py b/tests/unit/commands/completion/test_flag.py new file mode 100644 index 00000000..06e99979 --- /dev/null +++ b/tests/unit/commands/completion/test_flag.py @@ -0,0 +1,20 @@ +from borgmatic.commands.completion import flag as module + + +def test_variants_passes_through_non_list_index_flag_name(): + assert tuple(module.variants('foo')) == ('foo',) + + +def test_variants_broadcasts_list_index_flag_name_with_a_range_of_indices(): + assert tuple(module.variants('foo[0].bar')) == ( + 'foo[0].bar', + 'foo[1].bar', + 'foo[2].bar', + 'foo[3].bar', + 'foo[4].bar', + 'foo[5].bar', + 'foo[6].bar', + 'foo[7].bar', + 'foo[8].bar', + 'foo[9].bar', + ) diff --git a/tests/unit/commands/test_arguments.py b/tests/unit/commands/test_arguments.py index d10cf887..f3094277 100644 --- a/tests/unit/commands/test_arguments.py +++ b/tests/unit/commands/test_arguments.py @@ -575,3 +575,755 @@ def test_parse_arguments_for_actions_raises_error_when_no_action_is_specified(): with pytest.raises(ValueError): module.parse_arguments_for_actions(('config',), action_parsers, global_parser) + + +def test_make_argument_description_with_object_adds_example(): + buffer = flexmock() + buffer.should_receive('getvalue').and_return('{foo: example}') + flexmock(module.io).should_receive('StringIO').and_return(buffer) + yaml = flexmock() + yaml.should_receive('dump') + flexmock(module.ruamel.yaml).should_receive('YAML').and_return(yaml) + + assert ( + module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'object', + 'example': {'foo': 'example'}, + }, + flag_name='flag', + ) + == 'Thing. Example value: "{foo: example}"' + ) + + +def test_make_argument_description_without_description_and_with_object_sets_example(): + buffer = flexmock() + buffer.should_receive('getvalue').and_return('{foo: example}') + flexmock(module.io).should_receive('StringIO').and_return(buffer) + yaml = flexmock() + yaml.should_receive('dump') + flexmock(module.ruamel.yaml).should_receive('YAML').and_return(yaml) + + assert ( + module.make_argument_description( + schema={ + 'type': 'object', + 'example': {'foo': 'example'}, + }, + flag_name='flag', + ) + == 'Example value: "{foo: example}"' + ) + + +def test_make_argument_description_with_object_skips_missing_example(): + flexmock(module.ruamel.yaml).should_receive('YAML').never() + + assert ( + module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'object', + }, + flag_name='flag', + ) + == 'Thing.' + ) + + +def test_make_argument_description_with_array_adds_example(): + buffer = flexmock() + buffer.should_receive('getvalue').and_return('[example]') + flexmock(module.io).should_receive('StringIO').and_return(buffer) + yaml = flexmock() + yaml.should_receive('dump') + flexmock(module.ruamel.yaml).should_receive('YAML').and_return(yaml) + + assert ( + module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'array', + 'example': ['example'], + }, + flag_name='flag', + ) + == 'Thing. Example value: "[example]"' + ) + + +def test_make_argument_description_without_description_and_with_array_sets_example(): + buffer = flexmock() + buffer.should_receive('getvalue').and_return('[example]') + flexmock(module.io).should_receive('StringIO').and_return(buffer) + yaml = flexmock() + yaml.should_receive('dump') + flexmock(module.ruamel.yaml).should_receive('YAML').and_return(yaml) + + assert ( + module.make_argument_description( + schema={ + 'type': 'array', + 'example': ['example'], + }, + flag_name='flag', + ) + == 'Example value: "[example]"' + ) + + +def test_make_argument_description_with_array_skips_missing_example(): + flexmock(module.ruamel.yaml).should_receive('YAML').never() + + assert ( + module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'array', + }, + flag_name='flag', + ) + == 'Thing.' + ) + + +def test_make_argument_description_with_array_index_in_flag_name_adds_to_description(): + assert 'list element' in module.make_argument_description( + schema={ + 'description': 'Thing.', + 'type': 'something', + }, + flag_name='flag[0]', + ) + + +def test_make_argument_description_without_description_and_with_array_index_in_flag_name_sets_description(): + assert 'list element' in module.make_argument_description( + schema={ + 'type': 'something', + }, + flag_name='flag[0]', + ) + + +def test_make_argument_description_escapes_percent_character(): + assert ( + module.make_argument_description( + schema={ + 'description': '% Thing.', + 'type': 'something', + }, + flag_name='flag', + ) + == '%% Thing.' + ) + + +def test_add_array_element_arguments_without_array_index_bails(): + arguments_group = flexmock() + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=(), + flag_name='foo', + ) + + +def test_add_array_element_arguments_with_help_flag_bails(): + arguments_group = flexmock() + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo', '--help', '--bar'), + flag_name='foo[0]', + ) + + +def test_add_array_element_arguments_without_any_flags_bails(): + arguments_group = flexmock() + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=(), + flag_name='foo[0]', + ) + + +# Use this instead of a flexmock because it's not easy to check the type() of a flexmock instance. +Group_action = collections.namedtuple( + 'Group_action', + ( + 'option_strings', + 'choices', + 'default', + 'nargs', + 'required', + 'type', + ), + defaults=( + flexmock(), + flexmock(), + flexmock(), + flexmock(), + flexmock(), + ), +) + + +def test_add_array_element_arguments_without_array_index_flags_bails(): + arguments_group = flexmock( + _group_actions=( + Group_action( + option_strings=('--foo[0].val',), + ), + ), + _registries={'action': {'store_stuff': Group_action}}, + ) + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo', '--bar'), + flag_name='foo[0].val', + ) + + +def test_add_array_element_arguments_with_non_matching_array_index_flags_bails(): + arguments_group = flexmock( + _group_actions=( + Group_action( + option_strings=('--foo[0].val',), + ), + ), + _registries={'action': {'store_stuff': Group_action}}, + ) + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo', '--bar[25].val', 'barval'), + flag_name='foo[0].val', + ) + + +def test_add_array_element_arguments_with_identical_array_index_flag_bails(): + arguments_group = flexmock( + _group_actions=( + Group_action( + option_strings=('--foo[0].val',), + ), + ), + _registries={'action': {'store_stuff': Group_action}}, + ) + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo[0].val', 'fooval', '--bar'), + flag_name='foo[0].val', + ) + + +def test_add_array_element_arguments_without_action_type_in_registry_bails(): + arguments_group = flexmock( + _group_actions=( + Group_action( + option_strings=('--foo[0].val',), + choices=flexmock(), + default=flexmock(), + nargs=flexmock(), + required=flexmock(), + type=flexmock(), + ), + ), + _registries={'action': {'store_stuff': bool}}, + ) + arguments_group.should_receive('add_argument').never() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo[25].val', 'fooval', '--bar[1].val', 'barval'), + flag_name='foo[0].val', + ) + + +def test_add_array_element_arguments_adds_arguments_for_array_index_flags(): + arguments_group = flexmock( + _group_actions=( + Group_action( + option_strings=('--foo[0].val',), + choices=flexmock(), + default=flexmock(), + nargs=flexmock(), + required=flexmock(), + type=flexmock(), + ), + ), + _registries={'action': {'store_stuff': Group_action}}, + ) + arguments_group.should_receive('add_argument').with_args( + '--foo[25].val', + action='store_stuff', + choices=object, + default=object, + dest='foo[25].val', + nargs=object, + required=object, + type=object, + ).once() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo[25].val', 'fooval', '--bar[1].val', 'barval'), + flag_name='foo[0].val', + ) + + +def test_add_array_element_arguments_adds_arguments_for_array_index_flags_with_equals_sign(): + arguments_group = flexmock( + _group_actions=( + Group_action( + option_strings=('--foo[0].val',), + choices=flexmock(), + default=flexmock(), + nargs=flexmock(), + required=flexmock(), + type=flexmock(), + ), + ), + _registries={'action': {'store_stuff': Group_action}}, + ) + arguments_group.should_receive('add_argument').with_args( + '--foo[25].val', + action='store_stuff', + choices=object, + default=object, + dest='foo[25].val', + nargs=object, + required=object, + type=object, + ).once() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo[25].val=fooval', '--bar[1].val=barval'), + flag_name='foo[0].val', + ) + + +def test_add_array_element_arguments_adds_arguments_for_array_index_flags_with_dashes(): + arguments_group = flexmock( + _group_actions=( + Group_action( + option_strings=('--foo[0].val-and-stuff',), + choices=flexmock(), + default=flexmock(), + nargs=flexmock(), + required=flexmock(), + type=flexmock(), + ), + ), + _registries={'action': {'store_stuff': Group_action}}, + ) + arguments_group.should_receive('add_argument').with_args( + '--foo[25].val-and-stuff', + action='store_stuff', + choices=object, + default=object, + dest='foo[25].val_and_stuff', + nargs=object, + required=object, + type=object, + ).once() + + module.add_array_element_arguments( + arguments_group=arguments_group, + unparsed_arguments=('--foo[25].val-and-stuff', 'fooval', '--bar[1].val', 'barval'), + flag_name='foo[0].val-and-stuff', + ) + + +def test_add_arguments_from_schema_with_non_dict_schema_bails(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').never() + flexmock(module.borgmatic.config.schema).should_receive('parse_type').never() + arguments_group.should_receive('add_argument').never() + + module.add_arguments_from_schema( + arguments_group=arguments_group, schema='foo', unparsed_arguments=() + ) + + +def test_add_arguments_from_schema_with_nested_object_adds_flag_for_each_option(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help 1').and_return( + 'help 2' + ) + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return( + int + ).and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--foo.bar', + type=int, + metavar='BAR', + help='help 1', + ).once() + arguments_group.should_receive('add_argument').with_args( + '--foo.baz', + type=str, + metavar='BAZ', + help='help 2', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'object', + 'properties': { + 'bar': {'type': 'integer'}, + 'baz': {'type': 'str'}, + }, + } + }, + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_uses_first_non_null_type_from_multi_type_object(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help 1') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(int) + arguments_group.should_receive('add_argument').with_args( + '--foo.bar', + type=int, + metavar='BAR', + help='help 1', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': ['null', 'object', 'boolean'], + 'properties': { + 'bar': {'type': 'integer'}, + }, + } + }, + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_empty_multi_type_raises(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help 1') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(int) + arguments_group.should_receive('add_argument').never() + flexmock(module).should_receive('add_array_element_arguments').never() + + with pytest.raises(ValueError): + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': [], + 'properties': { + 'bar': {'type': 'integer'}, + }, + } + }, + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_propertyless_option_adds_flag(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--foo', + type=str, + metavar='FOO', + help='help', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'object', + } + }, + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_array_of_scalars_adds_multiple_flags(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').with_args( + 'integer', object=str, array=str + ).and_return(int) + flexmock(module.borgmatic.config.schema).should_receive('parse_type').with_args( + 'array', object=str, array=str + ).and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--foo[0]', + type=int, + metavar='FOO[0]', + help='help', + ).once() + arguments_group.should_receive('add_argument').with_args( + '--foo', + type=str, + metavar='FOO', + help='help', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'array', + 'items': { + 'type': 'integer', + }, + } + }, + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_array_of_objects_adds_multiple_flags(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help 1').and_return( + 'help 2' + ) + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return( + int + ).and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--foo[0].bar', + type=int, + metavar='BAR', + help='help 1', + ).once() + arguments_group.should_receive('add_argument').with_args( + '--foo', + type=str, + metavar='FOO', + help='help 2', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + flexmock(module).should_receive('add_array_element_arguments').with_args( + arguments_group=arguments_group, + unparsed_arguments=(), + flag_name='foo[0].bar', + ).once() + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'bar': { + 'type': 'integer', + } + }, + }, + } + }, + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_boolean_adds_two_valueless_flags(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(bool) + arguments_group.should_receive('add_argument').with_args( + '--foo', + action='store_true', + default=None, + help='help', + ).once() + arguments_group.should_receive('add_argument').with_args( + '--no-foo', + dest='foo', + action='store_false', + default=None, + help=object, + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'boolean', + } + }, + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_with_nested_boolean_adds_two_valueless_flags(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(bool) + arguments_group.should_receive('add_argument').with_args( + '--foo.bar.baz-quux', + action='store_true', + default=None, + help='help', + ).once() + arguments_group.should_receive('add_argument').with_args( + '--foo.bar.no-baz-quux', + dest='foo.bar.baz_quux', + action='store_false', + default=None, + help=object, + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'baz_quux': { + 'type': 'boolean', + } + }, + }, + unparsed_arguments=(), + names=('foo', 'bar'), + ) + + +def test_add_arguments_from_schema_with_boolean_with_name_prefixed_with_no_adds_two_valueless_flags_and_removes_the_no_for_one(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(bool) + arguments_group.should_receive('add_argument').with_args( + '--no-foo', + action='store_true', + default=None, + help='help', + ).once() + arguments_group.should_receive('add_argument').with_args( + '--foo', + dest='no_foo', + action='store_false', + default=None, + help=object, + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'no_foo': { + 'type': 'boolean', + } + }, + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_skips_omitted_flag_name(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--match-archives', + type=object, + metavar=object, + help=object, + ).never() + arguments_group.should_receive('add_argument').with_args( + '--foo', + type=str, + metavar='FOO', + help='help', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'match_archives': { + 'type': 'string', + }, + 'foo': { + 'type': 'string', + }, + }, + }, + unparsed_arguments=(), + ) + + +def test_add_arguments_from_schema_rewrites_option_name_to_flag_name(): + arguments_group = flexmock() + flexmock(module).should_receive('make_argument_description').and_return('help') + flexmock(module.borgmatic.config.schema).should_receive('parse_type').and_return(str) + arguments_group.should_receive('add_argument').with_args( + '--foo-and-stuff', + type=str, + metavar='FOO_AND_STUFF', + help='help', + ).once() + flexmock(module).should_receive('add_array_element_arguments') + + module.add_arguments_from_schema( + arguments_group=arguments_group, + schema={ + 'type': 'object', + 'properties': { + 'foo_and_stuff': { + 'type': 'string', + }, + }, + }, + unparsed_arguments=(), + ) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index e5e599e1..57f3cf67 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -27,9 +27,393 @@ def test_get_skip_actions_uses_config_and_arguments(config, arguments, expected_ assert module.get_skip_actions(config, arguments) == expected_actions +def test_monitoring_hooks_with_monioring_disabled_bails(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(module.DISABLED) + flexmock(module.dispatch).should_receive('call_hooks').never() + + with module.Monitoring_hooks( + config_filename='test.yaml', + config={}, + arguments={'create': flexmock()}, + global_arguments=flexmock(monitoring_verbosity=99, dry_run=False), + ): + pass + + +def test_monitoring_hooks_with_non_primary_action_bails(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(flexmock()) + flexmock(module.dispatch).should_receive('call_hooks').never() + + with module.Monitoring_hooks( + config_filename='test.yaml', + config={}, + arguments={'extract': flexmock()}, + global_arguments=flexmock(monitoring_verbosity=99, dry_run=False), + ): + pass + + +def test_monitoring_hooks_pings_monitors(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(flexmock()) + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'initialize_monitor', + object, + object, + object, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.START, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.LOG, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.FINISH, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.FAIL, + object, + object, + ).never() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'destroy_monitor', + object, + object, + object, + object, + ).once() + + with module.Monitoring_hooks( + config_filename='test.yaml', + config={}, + arguments={'create': flexmock()}, + global_arguments=flexmock(monitoring_verbosity=99, dry_run=False), + ): + pass + + +def test_monitoring_hooks_with_start_ping_error_raises(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(flexmock()) + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'initialize_monitor', + object, + object, + object, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.START, + object, + object, + ).and_raise(OSError).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.LOG, + object, + object, + ).never() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.FINISH, + object, + object, + ).never() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'destroy_monitor', + object, + object, + object, + object, + ).never() + + with pytest.raises(ValueError): + with module.Monitoring_hooks( + config_filename='test.yaml', + config={}, + arguments={'create': flexmock()}, + global_arguments=flexmock(monitoring_verbosity=99, dry_run=False), + ): + assert False # This should never get called. + + +def test_monitoring_hooks_with_log_ping_error_raises(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(flexmock()) + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'initialize_monitor', + object, + object, + object, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.START, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.LOG, + object, + object, + ).and_raise(OSError).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.FINISH, + object, + object, + ).never() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'destroy_monitor', + object, + object, + object, + object, + ).never() + + with pytest.raises(ValueError): + with module.Monitoring_hooks( + config_filename='test.yaml', + config={}, + arguments={'create': flexmock()}, + global_arguments=flexmock(monitoring_verbosity=99, dry_run=False), + ): + pass + + +def test_monitoring_hooks_with_finish_ping_error_raises(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(flexmock()) + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'initialize_monitor', + object, + object, + object, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.START, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.LOG, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.FINISH, + object, + object, + ).and_raise(OSError).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'destroy_monitor', + object, + object, + object, + object, + ).never() + + with pytest.raises(ValueError): + with module.Monitoring_hooks( + config_filename='test.yaml', + config={}, + arguments={'create': flexmock()}, + global_arguments=flexmock(monitoring_verbosity=99, dry_run=False), + ): + pass + + +def test_monitoring_with_wrapped_code_error_pings_fail(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(flexmock()) + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'initialize_monitor', + object, + object, + object, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.START, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.LOG, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.FINISH, + object, + object, + ).never() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.FAIL, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'destroy_monitor', + object, + object, + object, + object, + ).once() + + with pytest.raises(OSError): + with module.Monitoring_hooks( + config_filename='test.yaml', + config={}, + arguments={'create': flexmock()}, + global_arguments=flexmock(monitoring_verbosity=99, dry_run=False), + ): + raise OSError() + + +def test_monitoring_with_fail_ping_error_raise_original_error(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(flexmock()) + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'initialize_monitor', + object, + object, + object, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.START, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.LOG, + object, + object, + ).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.FINISH, + object, + object, + ).never() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'ping_monitor', + object, + object, + object, + module.monitor.State.FAIL, + object, + object, + ).and_raise(OSError).once() + flexmock(module.dispatch).should_receive('call_hooks').with_args( + 'destroy_monitor', + object, + object, + object, + object, + ).never() + + with pytest.raises(OSError): + with module.Monitoring_hooks( + config_filename='test.yaml', + config={}, + arguments={'create': flexmock()}, + global_arguments=flexmock(monitoring_verbosity=99, dry_run=False), + ): + raise OSError() + + def test_run_configuration_runs_actions_for_each_repository(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) expected_results = [flexmock(), flexmock()] flexmock(module).should_receive('Log_prefix').and_return(flexmock()) @@ -37,7 +421,7 @@ def test_run_configuration_runs_actions_for_each_repository(): expected_results[1:] ) config = {'repositories': [{'path': 'foo'}, {'path': 'bar'}]} - arguments = {'global': flexmock(monitoring_verbosity=1)} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock())} results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -47,11 +431,13 @@ def test_run_configuration_runs_actions_for_each_repository(): def test_run_configuration_with_skip_actions_does_not_raise(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return(['compact']) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_return(flexmock()).and_return(flexmock()) config = {'repositories': [{'path': 'foo'}, {'path': 'bar'}], 'skip_actions': ['compact']} - arguments = {'global': flexmock(monitoring_verbosity=1)} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock())} list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -59,65 +445,34 @@ def test_run_configuration_with_skip_actions_does_not_raise(): def test_run_configuration_with_invalid_borg_version_errors(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_raise(ValueError) - flexmock(module.command).should_receive('execute_hook').never() - flexmock(module.dispatch).should_receive('call_hooks').never() flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').never() config = {'repositories': [{'path': 'foo'}]} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'prune': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'prune': flexmock(), + } list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) -def test_run_configuration_logs_monitor_start_error(): - flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) - flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.dispatch).should_receive('call_hooks').and_raise(OSError).and_return( - None - ).and_return(None).and_return(None) - expected_results = [flexmock()] - flexmock(module).should_receive('log_error_records').and_return(expected_results) - flexmock(module).should_receive('Log_prefix').and_return(flexmock()) - flexmock(module).should_receive('run_actions').never() - config = {'repositories': [{'path': 'foo'}]} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} - - results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) - - assert results == expected_results - - -def test_run_configuration_bails_for_monitor_start_soft_failure(): - flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) - flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') - flexmock(module.dispatch).should_receive('call_hooks').and_raise(error).and_return(None) - flexmock(module).should_receive('log_error_records').never() - flexmock(module).should_receive('Log_prefix').and_return(flexmock()) - flexmock(module).should_receive('run_actions').never() - config = {'repositories': [{'path': 'foo'}, {'path': 'bar'}]} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} - - results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) - - assert results == [] - - def test_run_configuration_logs_actions_error(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.command).should_receive('execute_hook') - flexmock(module.dispatch).should_receive('call_hooks') expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError) + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') config = {'repositories': [{'path': 'foo'}]} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False)} + arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock())} results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -127,8 +482,9 @@ def test_run_configuration_logs_actions_error(): def test_run_configuration_skips_remaining_actions_for_actions_soft_failure_but_still_runs_next_repository_actions(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.dispatch).should_receive('call_hooks').times(5) error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') log = flexmock() flexmock(module).should_receive('Log_prefix').and_return(flexmock()) @@ -136,102 +492,31 @@ def test_run_configuration_skips_remaining_actions_for_actions_soft_failure_but_ flexmock(module).should_receive('log_error_records').never() flexmock(module.command).should_receive('considered_soft_failure').and_return(True) config = {'repositories': [{'path': 'foo'}, {'path': 'bar'}]} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) assert results == [log] -def test_run_configuration_logs_monitor_log_error(): - flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) - flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return( - None - ).and_raise(OSError) - expected_results = [flexmock()] - flexmock(module).should_receive('log_error_records').and_return(expected_results) - flexmock(module).should_receive('Log_prefix').and_return(flexmock()) - flexmock(module).should_receive('run_actions').and_return([]) - config = {'repositories': [{'path': 'foo'}]} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} - - results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) - - assert results == expected_results - - -def test_run_configuration_still_pings_monitor_for_monitor_log_soft_failure(): - flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) - flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') - flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return( - None - ).and_raise(error).and_return(None).and_return(None).times(5) - flexmock(module).should_receive('log_error_records').never() - flexmock(module).should_receive('Log_prefix').and_return(flexmock()) - flexmock(module).should_receive('run_actions').and_return([]) - flexmock(module.command).should_receive('considered_soft_failure').and_return(True) - config = {'repositories': [{'path': 'foo'}]} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} - - results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) - - assert results == [] - - -def test_run_configuration_logs_monitor_finish_error(): - flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) - flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return( - None - ).and_return(None).and_raise(OSError) - expected_results = [flexmock()] - flexmock(module).should_receive('log_error_records').and_return(expected_results) - flexmock(module).should_receive('Log_prefix').and_return(flexmock()) - flexmock(module).should_receive('run_actions').and_return([]) - config = {'repositories': [{'path': 'foo'}]} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} - - results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) - - assert results == expected_results - - -def test_run_configuration_bails_for_monitor_finish_soft_failure(): - flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) - flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') - flexmock(module.dispatch).should_receive('call_hooks').and_return(None).and_return( - None - ).and_raise(None).and_raise(error) - flexmock(module).should_receive('log_error_records').never() - flexmock(module).should_receive('Log_prefix').and_return(flexmock()) - flexmock(module).should_receive('run_actions').and_return([]) - flexmock(module.command).should_receive('considered_soft_failure').and_return(True) - config = {'repositories': [{'path': 'foo'}]} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} - - results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) - - assert results == [] - - def test_run_configuration_does_not_call_monitoring_hooks_if_monitoring_hooks_are_disabled(): flexmock(module).should_receive('verbosity_to_log_level').and_return(module.DISABLED) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.dispatch).should_receive('call_hooks').never() flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_return([]) config = {'repositories': [{'path': 'foo'}]} - arguments = {'global': flexmock(monitoring_verbosity=-2, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=-2, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) assert results == [] @@ -239,8 +524,11 @@ def test_run_configuration_does_not_call_monitoring_hooks_if_monitoring_hooks_ar def test_run_configuration_logs_on_error_hook_error(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.command).should_receive('execute_hook').and_raise(OSError) + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks').and_raise(OSError) expected_results = [flexmock(), flexmock()] flexmock(module).should_receive('log_error_records').and_return( expected_results[:1] @@ -248,7 +536,56 @@ def test_run_configuration_logs_on_error_hook_error(): flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError) config = {'repositories': [{'path': 'foo'}]} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } + + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) + + assert results == expected_results + + +def test_run_configuration_logs_on_before_command_hook_error(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) + flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) + flexmock(module.command).should_receive('Before_after_hooks').and_raise(OSError) + flexmock(module.borg_version).should_receive('local_borg_version').never() + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') + expected_results = [flexmock()] + flexmock(module).should_receive('log_error_records').and_return(expected_results) + flexmock(module).should_receive('Log_prefix').never() + flexmock(module).should_receive('run_actions').never() + config = {'repositories': [{'path': 'foo'}]} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } + + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) + + assert results == expected_results + + +def test_run_configuration_logs_on_monitoring_hook_error(): + flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) + flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_raise(OSError) + flexmock(module.command).should_receive('Before_after_hooks').never() + flexmock(module.borg_version).should_receive('local_borg_version').never() + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') + expected_results = [flexmock()] + flexmock(module).should_receive('log_error_records').and_return(expected_results) + flexmock(module).should_receive('Log_prefix').never() + flexmock(module).should_receive('run_actions').never() + config = {'repositories': [{'path': 'foo'}]} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -258,15 +595,21 @@ def test_run_configuration_logs_on_error_hook_error(): def test_run_configuration_bails_for_on_error_hook_soft_failure(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') - flexmock(module.command).should_receive('execute_hook').and_raise(error) + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks').and_raise(error) expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError) config = {'repositories': [{'path': 'foo'}]} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) @@ -277,14 +620,22 @@ def test_run_configuration_retries_soft_error(): # Run action first fails, second passes. flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError).and_return([]) flexmock(module).should_receive('log_error_records').and_return([flexmock()]).once() + flexmock(module.command).should_receive('filter_hooks').never() + flexmock(module.command).should_receive('execute_hooks').never() config = {'repositories': [{'path': 'foo'}], 'retries': 1} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) + assert results == [] @@ -292,8 +643,9 @@ def test_run_configuration_retries_hard_error(): # Run action fails twice. flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError).times(2) flexmock(module).should_receive('log_error_records').with_args( @@ -307,17 +659,25 @@ def test_run_configuration_retries_hard_error(): 'Error running actions for repository', OSError, ).and_return(error_logs) + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') config = {'repositories': [{'path': 'foo'}], 'retries': 1} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) + assert results == error_logs -def test_run_configuration_repos_ordered(): +def test_run_configuration_retries_repositories_in_order(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError).times(2) expected_results = [flexmock(), flexmock()] @@ -327,17 +687,25 @@ def test_run_configuration_repos_ordered(): flexmock(module).should_receive('log_error_records').with_args( 'Error running actions for repository', OSError ).and_return(expected_results[1:]).ordered() + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') config = {'repositories': [{'path': 'foo'}, {'path': 'bar'}]} - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) + assert results == expected_results def test_run_configuration_retries_round_robin(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) flexmock(module).should_receive('log_error_records').with_args( @@ -360,20 +728,28 @@ def test_run_configuration_retries_round_robin(): flexmock(module).should_receive('log_error_records').with_args( 'Error running actions for repository', OSError ).and_return(bar_error_logs).ordered() + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') config = { 'repositories': [{'path': 'foo'}, {'path': 'bar'}], 'retries': 1, } - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) + assert results == foo_error_logs + bar_error_logs -def test_run_configuration_retries_one_passes(): +def test_run_configuration_with_one_retry(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return( [] @@ -394,20 +770,28 @@ def test_run_configuration_retries_one_passes(): flexmock(module).should_receive('log_error_records').with_args( 'Error running actions for repository', OSError ).and_return(error_logs).ordered() + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') config = { 'repositories': [{'path': 'foo'}, {'path': 'bar'}], 'retries': 1, } - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) + assert results == error_logs -def test_run_configuration_retry_wait(): +def test_run_configuration_with_retry_wait_does_backoff_after_each_retry(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) flexmock(module).should_receive('log_error_records').with_args( @@ -438,21 +822,29 @@ def test_run_configuration_retry_wait(): flexmock(module).should_receive('log_error_records').with_args( 'Error running actions for repository', OSError ).and_return(error_logs).ordered() + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') config = { 'repositories': [{'path': 'foo'}], 'retries': 3, 'retry_wait': 10, } - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) + assert results == error_logs -def test_run_configuration_retries_timeout_multiple_repos(): +def test_run_configuration_with_multiple_repositories_retries_with_timeout(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module).should_receive('Monitoring_hooks').and_return(flexmock()) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) - flexmock(module.command).should_receive('execute_hook') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return( [] @@ -479,20 +871,27 @@ def test_run_configuration_retries_timeout_multiple_repos(): flexmock(module).should_receive('log_error_records').with_args( 'Error running actions for repository', OSError ).and_return(error_logs).ordered() + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') config = { 'repositories': [{'path': 'foo'}, {'path': 'bar'}], 'retries': 1, 'retry_wait': 10, } - arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} + arguments = { + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + 'create': flexmock(), + } + results = list(module.run_configuration('test.yaml', config, ['/tmp/test.yaml'], arguments)) + assert results == error_logs def test_run_actions_runs_repo_create(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.repo_create).should_receive('run_repo_create').once() tuple( @@ -515,19 +914,13 @@ def test_run_actions_runs_repo_create(): def test_run_actions_adds_label_file_to_hook_context(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) expected = flexmock() flexmock(borgmatic.actions.create).should_receive('run_create').with_args( config_filename=object, repository={'path': 'repo', 'label': 'my repo'}, config={'repositories': []}, config_paths=[], - hook_context={ - 'repository_label': 'my repo', - 'log_file': '', - 'repositories': '', - 'repository': 'repo', - }, local_borg_version=object, create_arguments=object, global_arguments=object, @@ -554,19 +947,13 @@ def test_run_actions_adds_label_file_to_hook_context(): def test_run_actions_adds_log_file_to_hook_context(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) expected = flexmock() flexmock(borgmatic.actions.create).should_receive('run_create').with_args( config_filename=object, repository={'path': 'repo'}, config={'repositories': []}, config_paths=[], - hook_context={ - 'repository_label': '', - 'log_file': 'foo', - 'repositories': '', - 'repository': 'repo', - }, local_borg_version=object, create_arguments=object, global_arguments=object, @@ -593,7 +980,7 @@ def test_run_actions_adds_log_file_to_hook_context(): def test_run_actions_runs_transfer(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.transfer).should_receive('run_transfer').once() tuple( @@ -613,7 +1000,7 @@ def test_run_actions_runs_transfer(): def test_run_actions_runs_create(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) expected = flexmock() flexmock(borgmatic.actions.create).should_receive('run_create').and_yield(expected).once() @@ -635,7 +1022,7 @@ def test_run_actions_runs_create(): def test_run_actions_with_skip_actions_skips_create(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return(['create']) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.create).should_receive('run_create').never() tuple( @@ -652,10 +1039,51 @@ def test_run_actions_with_skip_actions_skips_create(): ) +def test_run_actions_runs_recreate(): + flexmock(module).should_receive('add_custom_log_levels') + flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) + + flexmock(borgmatic.actions.recreate).should_receive('run_recreate').once() + + tuple( + module.run_actions( + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'recreate': flexmock()}, + config_filename=flexmock(), + config={'repositories': []}, + config_paths=[], + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), + repository={'path': 'repo'}, + ) + ) + + +def test_run_actions_with_skip_actions_skips_recreate(): + flexmock(module).should_receive('add_custom_log_levels') + flexmock(module).should_receive('get_skip_actions').and_return(['recreate']) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) + flexmock(borgmatic.actions.recreate).should_receive('run_recreate').never() + + tuple( + module.run_actions( + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'recreate': flexmock()}, + config_filename=flexmock(), + config={'repositories': [], 'skip_actions': ['recreate']}, + config_paths=[], + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), + repository={'path': 'repo'}, + ) + ) + + def test_run_actions_runs_prune(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.prune).should_receive('run_prune').once() tuple( @@ -675,7 +1103,7 @@ def test_run_actions_runs_prune(): def test_run_actions_with_skip_actions_skips_prune(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return(['prune']) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.prune).should_receive('run_prune').never() tuple( @@ -695,7 +1123,7 @@ def test_run_actions_with_skip_actions_skips_prune(): def test_run_actions_runs_compact(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.compact).should_receive('run_compact').once() tuple( @@ -715,7 +1143,7 @@ def test_run_actions_runs_compact(): def test_run_actions_with_skip_actions_skips_compact(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return(['compact']) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.compact).should_receive('run_compact').never() tuple( @@ -735,7 +1163,7 @@ def test_run_actions_with_skip_actions_skips_compact(): def test_run_actions_runs_check_when_repository_enabled_for_checks(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.checks).should_receive('repository_enabled_for_checks').and_return(True) flexmock(borgmatic.actions.check).should_receive('run_check').once() @@ -756,7 +1184,7 @@ def test_run_actions_runs_check_when_repository_enabled_for_checks(): def test_run_actions_skips_check_when_repository_not_enabled_for_checks(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.checks).should_receive('repository_enabled_for_checks').and_return(False) flexmock(borgmatic.actions.check).should_receive('run_check').never() @@ -777,7 +1205,7 @@ def test_run_actions_skips_check_when_repository_not_enabled_for_checks(): def test_run_actions_with_skip_actions_skips_check(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return(['check']) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(module.checks).should_receive('repository_enabled_for_checks').and_return(True) flexmock(borgmatic.actions.check).should_receive('run_check').never() @@ -798,7 +1226,7 @@ def test_run_actions_with_skip_actions_skips_check(): def test_run_actions_runs_extract(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.extract).should_receive('run_extract').once() tuple( @@ -818,7 +1246,7 @@ def test_run_actions_runs_extract(): def test_run_actions_runs_export_tar(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.export_tar).should_receive('run_export_tar').once() tuple( @@ -838,7 +1266,7 @@ def test_run_actions_runs_export_tar(): def test_run_actions_runs_mount(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.mount).should_receive('run_mount').once() tuple( @@ -858,7 +1286,7 @@ def test_run_actions_runs_mount(): def test_run_actions_runs_restore(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.restore).should_receive('run_restore').once() tuple( @@ -878,7 +1306,7 @@ def test_run_actions_runs_restore(): def test_run_actions_runs_repo_list(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) expected = flexmock() flexmock(borgmatic.actions.repo_list).should_receive('run_repo_list').and_yield(expected).once() @@ -900,7 +1328,7 @@ def test_run_actions_runs_repo_list(): def test_run_actions_runs_list(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) expected = flexmock() flexmock(borgmatic.actions.list).should_receive('run_list').and_yield(expected).once() @@ -922,7 +1350,7 @@ def test_run_actions_runs_list(): def test_run_actions_runs_repo_info(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) expected = flexmock() flexmock(borgmatic.actions.repo_info).should_receive('run_repo_info').and_yield(expected).once() @@ -944,7 +1372,7 @@ def test_run_actions_runs_repo_info(): def test_run_actions_runs_info(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) expected = flexmock() flexmock(borgmatic.actions.info).should_receive('run_info').and_yield(expected).once() @@ -966,7 +1394,7 @@ def test_run_actions_runs_info(): def test_run_actions_runs_break_lock(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.break_lock).should_receive('run_break_lock').once() tuple( @@ -986,7 +1414,7 @@ def test_run_actions_runs_break_lock(): def test_run_actions_runs_export_key(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.export_key).should_receive('run_export_key').once() tuple( @@ -1003,10 +1431,30 @@ def test_run_actions_runs_export_key(): ) +def test_run_actions_runs_import_key(): + flexmock(module).should_receive('add_custom_log_levels') + flexmock(module).should_receive('get_skip_actions').and_return([]) + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) + flexmock(borgmatic.actions.import_key).should_receive('run_import_key').once() + + tuple( + module.run_actions( + arguments={'global': flexmock(dry_run=False, log_file='foo'), 'import': flexmock()}, + config_filename=flexmock(), + config={'repositories': []}, + config_paths=[], + local_path=flexmock(), + remote_path=flexmock(), + local_borg_version=flexmock(), + repository={'path': 'repo'}, + ) + ) + + def test_run_actions_runs_change_passphrase(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.change_passphrase).should_receive('run_change_passphrase').once() tuple( @@ -1029,7 +1477,7 @@ def test_run_actions_runs_change_passphrase(): def test_run_actions_runs_delete(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.delete).should_receive('run_delete').once() tuple( @@ -1049,7 +1497,7 @@ def test_run_actions_runs_delete(): def test_run_actions_runs_repo_delete(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.repo_delete).should_receive('run_repo_delete').once() tuple( @@ -1072,7 +1520,7 @@ def test_run_actions_runs_repo_delete(): def test_run_actions_runs_borg(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.borg).should_receive('run_borg').once() tuple( @@ -1092,7 +1540,7 @@ def test_run_actions_runs_borg(): def test_run_actions_runs_multiple_actions_in_argument_order(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module).should_receive('get_skip_actions').and_return([]) - flexmock(module.command).should_receive('execute_hook') + flexmock(module.command).should_receive('Before_after_hooks').and_return(flexmock()) flexmock(borgmatic.actions.borg).should_receive('run_borg').once().ordered() flexmock(borgmatic.actions.restore).should_receive('run_restore').once().ordered() @@ -1114,7 +1562,11 @@ def test_run_actions_runs_multiple_actions_in_argument_order(): ) -def test_load_configurations_collects_parsed_configurations_and_logs(): +@pytest.mark.parametrize( + 'resolve_env', + ((True, False),), +) +def test_load_configurations_collects_parsed_configurations_and_logs(resolve_env): configuration = flexmock() other_configuration = flexmock() test_expected_logs = [flexmock(), flexmock()] @@ -1123,7 +1575,13 @@ def test_load_configurations_collects_parsed_configurations_and_logs(): configuration, ['/tmp/test.yaml'], test_expected_logs ).and_return(other_configuration, ['/tmp/other.yaml'], other_expected_logs) - configs, config_paths, logs = tuple(module.load_configurations(('test.yaml', 'other.yaml'))) + configs, config_paths, logs = tuple( + module.load_configurations( + ('test.yaml', 'other.yaml'), + arguments=flexmock(), + resolve_env=resolve_env, + ) + ) assert configs == {'test.yaml': configuration, 'other.yaml': other_configuration} assert config_paths == ['/tmp/other.yaml', '/tmp/test.yaml'] @@ -1133,7 +1591,9 @@ def test_load_configurations_collects_parsed_configurations_and_logs(): def test_load_configurations_logs_warning_for_permission_error(): flexmock(module.validate).should_receive('parse_configuration').and_raise(PermissionError) - configs, config_paths, logs = tuple(module.load_configurations(('test.yaml',))) + configs, config_paths, logs = tuple( + module.load_configurations(('test.yaml',), arguments=flexmock()) + ) assert configs == {} assert config_paths == [] @@ -1143,7 +1603,9 @@ def test_load_configurations_logs_warning_for_permission_error(): def test_load_configurations_logs_critical_for_parse_error(): flexmock(module.validate).should_receive('parse_configuration').and_raise(ValueError) - configs, config_paths, logs = tuple(module.load_configurations(('test.yaml',))) + configs, config_paths, logs = tuple( + module.load_configurations(('test.yaml',), arguments=flexmock()) + ) assert configs == {} assert config_paths == [] @@ -1389,11 +1851,12 @@ def test_collect_highlander_action_summary_logs_error_on_run_validate_failure(): def test_collect_configuration_run_summary_logs_info_for_success(): - flexmock(module.command).should_receive('execute_hook').never() flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return([]) - arguments = {} + arguments = {'global': flexmock(dry_run=False, log_file=flexmock())} logs = tuple( module.collect_configuration_run_summary_logs( @@ -1406,9 +1869,14 @@ def test_collect_configuration_run_summary_logs_info_for_success(): def test_collect_configuration_run_summary_executes_hooks_for_create(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return([]) - arguments = {'create': flexmock(), 'global': flexmock(monitoring_verbosity=1, dry_run=False)} + arguments = { + 'create': flexmock(), + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + } logs = tuple( module.collect_configuration_run_summary_logs( @@ -1421,9 +1889,14 @@ def test_collect_configuration_run_summary_executes_hooks_for_create(): def test_collect_configuration_run_summary_logs_info_for_success_with_extract(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return([]) - arguments = {'extract': flexmock(repository='repo')} + arguments = { + 'extract': flexmock(repository='repo'), + 'global': flexmock(dry_run=False, log_file=flexmock()), + } logs = tuple( module.collect_configuration_run_summary_logs( @@ -1440,7 +1913,7 @@ def test_collect_configuration_run_summary_logs_extract_with_repository_error(): ) expected_logs = (flexmock(),) flexmock(module).should_receive('log_error_records').and_return(expected_logs) - arguments = {'extract': flexmock(repository='repo')} + arguments = {'extract': flexmock(repository='repo', log_file=flexmock())} logs = tuple( module.collect_configuration_run_summary_logs( @@ -1453,9 +1926,14 @@ def test_collect_configuration_run_summary_logs_extract_with_repository_error(): def test_collect_configuration_run_summary_logs_info_for_success_with_mount(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return([]) - arguments = {'mount': flexmock(repository='repo')} + arguments = { + 'mount': flexmock(repository='repo'), + 'global': flexmock(dry_run=False, log_file=flexmock()), + } logs = tuple( module.collect_configuration_run_summary_logs( @@ -1472,7 +1950,10 @@ def test_collect_configuration_run_summary_logs_mount_with_repository_error(): ) expected_logs = (flexmock(),) flexmock(module).should_receive('log_error_records').and_return(expected_logs) - arguments = {'mount': flexmock(repository='repo')} + arguments = { + 'mount': flexmock(repository='repo'), + 'global': flexmock(dry_run=False, log_file=flexmock()), + } logs = tuple( module.collect_configuration_run_summary_logs( @@ -1484,7 +1965,10 @@ def test_collect_configuration_run_summary_logs_mount_with_repository_error(): def test_collect_configuration_run_summary_logs_missing_configs_error(): - arguments = {'global': flexmock(config_paths=[])} + flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') + arguments = {'global': flexmock(config_paths=[], log_file=flexmock())} expected_logs = (flexmock(),) flexmock(module).should_receive('log_error_records').and_return(expected_logs) @@ -1496,10 +1980,15 @@ def test_collect_configuration_run_summary_logs_missing_configs_error(): def test_collect_configuration_run_summary_logs_pre_hook_error(): - flexmock(module.command).should_receive('execute_hook').and_raise(ValueError) + flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks').and_raise(ValueError) expected_logs = (flexmock(),) flexmock(module).should_receive('log_error_records').and_return(expected_logs) - arguments = {'create': flexmock(), 'global': flexmock(monitoring_verbosity=1, dry_run=False)} + arguments = { + 'create': flexmock(), + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + } logs = tuple( module.collect_configuration_run_summary_logs( @@ -1511,13 +2000,17 @@ def test_collect_configuration_run_summary_logs_pre_hook_error(): def test_collect_configuration_run_summary_logs_post_hook_error(): - flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise(ValueError) flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks').and_return(None).and_raise(ValueError) flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return([]) expected_logs = (flexmock(),) flexmock(module).should_receive('log_error_records').and_return(expected_logs) - arguments = {'create': flexmock(), 'global': flexmock(monitoring_verbosity=1, dry_run=False)} + arguments = { + 'create': flexmock(), + 'global': flexmock(monitoring_verbosity=1, dry_run=False, log_file=flexmock()), + } logs = tuple( module.collect_configuration_run_summary_logs( @@ -1534,7 +2027,10 @@ def test_collect_configuration_run_summary_logs_for_list_with_archive_and_reposi ) expected_logs = (flexmock(),) flexmock(module).should_receive('log_error_records').and_return(expected_logs) - arguments = {'list': flexmock(repository='repo', archive='test')} + arguments = { + 'list': flexmock(repository='repo', archive='test'), + 'global': flexmock(dry_run=False, log_file=flexmock()), + } logs = tuple( module.collect_configuration_run_summary_logs( @@ -1547,9 +2043,14 @@ def test_collect_configuration_run_summary_logs_for_list_with_archive_and_reposi def test_collect_configuration_run_summary_logs_info_for_success_with_list(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return([]) - arguments = {'list': flexmock(repository='repo', archive=None)} + arguments = { + 'list': flexmock(repository='repo', archive=None), + 'global': flexmock(dry_run=False, log_file=flexmock()), + } logs = tuple( module.collect_configuration_run_summary_logs( @@ -1560,14 +2061,16 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_list(): assert {log.levelno for log in logs} == {logging.INFO} -def test_collect_configuration_run_summary_logs_run_configuration_error(): +def test_collect_configuration_run_summary_logs_run_configuration_error_logs(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return( [logging.makeLogRecord(dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg='Error'))] ) flexmock(module).should_receive('log_error_records').and_return([]) - arguments = {} + arguments = {'global': flexmock(dry_run=False, log_file=flexmock())} logs = tuple( module.collect_configuration_run_summary_logs( @@ -1580,13 +2083,18 @@ def test_collect_configuration_run_summary_logs_run_configuration_error(): def test_collect_configuration_run_summary_logs_run_umount_error(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return([]) flexmock(module.borg_umount).should_receive('unmount_archive').and_raise(OSError) flexmock(module).should_receive('log_error_records').and_return( [logging.makeLogRecord(dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg='Error'))] ) - arguments = {'umount': flexmock(mount_point='/mnt')} + arguments = { + 'umount': flexmock(mount_point='/mnt'), + 'global': flexmock(dry_run=False, log_file=flexmock()), + } logs = tuple( module.collect_configuration_run_summary_logs( @@ -1599,6 +2107,8 @@ def test_collect_configuration_run_summary_logs_run_umount_error(): def test_collect_configuration_run_summary_logs_outputs_merged_json_results(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module.command).should_receive('filter_hooks') + flexmock(module.command).should_receive('execute_hooks') flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return(['foo', 'bar']).and_return( ['baz'] @@ -1606,7 +2116,7 @@ def test_collect_configuration_run_summary_logs_outputs_merged_json_results(): stdout = flexmock() stdout.should_receive('write').with_args('["foo", "bar", "baz"]').once() flexmock(module.sys).stdout = stdout - arguments = {} + arguments = {'global': flexmock(dry_run=False, log_file=flexmock())} tuple( module.collect_configuration_run_summary_logs( @@ -1615,3 +2125,56 @@ def test_collect_configuration_run_summary_logs_outputs_merged_json_results(): arguments=arguments, ) ) + + +def test_check_and_show_help_on_no_args_shows_help_when_no_args_and_default_actions_false(): + flexmock(module.sys).should_receive('argv').and_return(['borgmatic']) + flexmock(module).should_receive('parse_arguments').with_args('--help').once() + flexmock(module.sys).should_receive('exit').with_args(0).once() + module.check_and_show_help_on_no_args({'test.yaml': {'default_actions': False}}) + + +def test_check_and_show_help_on_no_args_does_not_show_help_when_no_args_and_default_actions_true(): + flexmock(module.sys).should_receive('argv').and_return(['borgmatic']) + flexmock(module).should_receive('parse_arguments').never() + flexmock(module.sys).should_receive('exit').never() + module.check_and_show_help_on_no_args({'test.yaml': {'default_actions': True}}) + + +def test_check_and_show_help_on_no_args_does_not_show_help_when_args_provided(): + flexmock(module.sys).should_receive('argv').and_return(['borgmatic', '--create']) + flexmock(module).should_receive('parse_arguments').never() + flexmock(module.sys).should_receive('exit').never() + module.check_and_show_help_on_no_args({'test.yaml': {'default_actions': False}}) + + +def test_check_and_show_help_on_no_args_with_no_default_actions_in_all_configs(): + flexmock(module.sys).should_receive('argv').and_return(['borgmatic']) + + # Both configs have default_actions set to False, so help should be shown + configs = { + 'config1.yaml': {'default_actions': False}, + 'config2.yaml': {'default_actions': False}, + } + + # Expect help to be shown + flexmock(module).should_receive('parse_arguments').with_args('--help').once() + flexmock(module.sys).should_receive('exit').with_args(0).once() + + module.check_and_show_help_on_no_args(configs) + + +def test_check_and_show_help_on_no_args_with_conflicting_configs(): + flexmock(module.sys).should_receive('argv').and_return(['borgmatic']) + + # Simulate two config files with conflicting 'default_actions' values + configs = { + 'config1.yaml': {'default_actions': True}, + 'config2.yaml': {'default_actions': False}, + } + + # Expect help not to be shown because at least one config enables default actions + flexmock(module).should_receive('parse_arguments').never() + flexmock(module.sys).should_receive('exit').never() + + module.check_and_show_help_on_no_args(configs) diff --git a/tests/unit/config/test_arguments.py b/tests/unit/config/test_arguments.py new file mode 100644 index 00000000..fdf5f0f5 --- /dev/null +++ b/tests/unit/config/test_arguments.py @@ -0,0 +1,234 @@ +import pytest +from flexmock import flexmock + +from borgmatic.config import arguments as module + + +def test_set_values_without_keys_bails(): + config = {'option': 'value'} + module.set_values(config=config, keys=(), value=5) + + assert config == {'option': 'value'} + + +def test_set_values_with_keys_adds_them_to_config(): + config = {'option': 'value'} + + module.set_values(config=config, keys=('foo', 'bar', 'baz'), value=5) + + assert config == {'option': 'value', 'foo': {'bar': {'baz': 5}}} + + +def test_set_values_with_one_existing_key_adds_others_to_config(): + config = {'foo': {'other': 'value'}} + + module.set_values(config=config, keys=('foo', 'bar', 'baz'), value=5) + + assert config == {'foo': {'other': 'value', 'bar': {'baz': 5}}} + + +def test_set_values_with_two_existing_keys_adds_others_to_config(): + config = {'foo': {'bar': {'other': 'value'}}} + + module.set_values(config=config, keys=('foo', 'bar', 'baz'), value=5) + + assert config == {'foo': {'bar': {'other': 'value', 'baz': 5}}} + + +def test_set_values_with_list_index_key_adds_it_to_config(): + config = {'foo': {'bar': [{'option': 'value'}, {'other': 'thing'}]}} + + module.set_values(config=config, keys=('foo', 'bar[1]', 'baz'), value=5) + + assert config == {'foo': {'bar': [{'option': 'value'}, {'other': 'thing', 'baz': 5}]}} + + +def test_set_values_with_list_index_key_out_of_range_raises(): + config = {'foo': {'bar': [{'option': 'value'}]}} + + with pytest.raises(ValueError): + module.set_values(config=config, keys=('foo', 'bar[1]', 'baz'), value=5) + + +def test_set_values_with_final_list_index_key_out_of_range_raises(): + config = {'foo': {'bar': [{'option': 'value'}]}} + + with pytest.raises(ValueError): + module.set_values(config=config, keys=('foo', 'bar[1]'), value=5) + + +def test_set_values_with_list_index_key_missing_list_and_out_of_range_raises(): + config = {'other': 'value'} + + with pytest.raises(ValueError): + module.set_values(config=config, keys=('foo', 'bar[1]', 'baz'), value=5) + + +def test_set_values_with_final_list_index_key_adds_it_to_config(): + config = {'foo': {'bar': [1, 2]}} + + module.set_values(config=config, keys=('foo', 'bar[1]'), value=5) + + assert config == {'foo': {'bar': [1, 5]}} + + +def test_type_for_option_with_option_finds_type(): + flexmock(module.borgmatic.config.schema).should_receive('get_properties').replace_with( + lambda sub_schema: sub_schema['properties'] + ) + + assert ( + module.type_for_option( + schema={'type': 'object', 'properties': {'foo': {'type': 'integer'}}}, + option_keys=('foo',), + ) + == 'integer' + ) + + +def test_type_for_option_with_nested_option_finds_type(): + flexmock(module.borgmatic.config.schema).should_receive('get_properties').replace_with( + lambda sub_schema: sub_schema['properties'] + ) + + assert ( + module.type_for_option( + schema={ + 'type': 'object', + 'properties': { + 'foo': {'type': 'object', 'properties': {'bar': {'type': 'boolean'}}} + }, + }, + option_keys=('foo', 'bar'), + ) + == 'boolean' + ) + + +def test_type_for_option_with_missing_nested_option_finds_nothing(): + flexmock(module.borgmatic.config.schema).should_receive('get_properties').replace_with( + lambda sub_schema: sub_schema['properties'] + ) + + assert ( + module.type_for_option( + schema={ + 'type': 'object', + 'properties': { + 'foo': {'type': 'object', 'properties': {'other': {'type': 'integer'}}} + }, + }, + option_keys=('foo', 'bar'), + ) + is None + ) + + +def test_type_for_option_with_typeless_nested_option_finds_nothing(): + flexmock(module.borgmatic.config.schema).should_receive('get_properties').replace_with( + lambda sub_schema: sub_schema['properties'] + ) + + assert ( + module.type_for_option( + schema={ + 'type': 'object', + 'properties': {'foo': {'type': 'object', 'properties': {'bar': {'example': 5}}}}, + }, + option_keys=('foo', 'bar'), + ) + is None + ) + + +def test_type_for_option_with_list_index_option_finds_type(): + flexmock(module.borgmatic.config.schema).should_receive('get_properties').replace_with( + lambda sub_schema: sub_schema['properties'] + ) + + assert ( + module.type_for_option( + schema={ + 'type': 'object', + 'properties': {'foo': {'type': 'array', 'items': {'type': 'integer'}}}, + }, + option_keys=('foo[0]',), + ) + == 'integer' + ) + + +def test_type_for_option_with_nested_list_index_option_finds_type(): + flexmock(module.borgmatic.config.schema).should_receive('get_properties').replace_with( + lambda sub_schema: sub_schema['properties'] + ) + + assert ( + module.type_for_option( + schema={ + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'array', + 'items': {'type': 'object', 'properties': {'bar': {'type': 'integer'}}}, + } + }, + }, + option_keys=('foo[0]', 'bar'), + ) + == 'integer' + ) + + +def test_prepare_arguments_for_config_converts_arguments_to_keys(): + assert module.prepare_arguments_for_config( + global_arguments=flexmock(**{'my_option.sub_option': 'value1', 'other_option': 'value2'}), + schema={ + 'type': 'object', + 'properties': { + 'my_option': {'type': 'object', 'properties': {'sub_option': {'type': 'string'}}}, + 'other_option': {'type': 'string'}, + }, + }, + ) == ( + (('my_option', 'sub_option'), 'value1'), + (('other_option',), 'value2'), + ) + + +def test_prepare_arguments_for_config_skips_option_with_none_value(): + assert module.prepare_arguments_for_config( + global_arguments=flexmock(**{'my_option.sub_option': None, 'other_option': 'value2'}), + schema={ + 'type': 'object', + 'properties': { + 'my_option': {'type': 'object', 'properties': {'sub_option': {'type': 'string'}}}, + 'other_option': {'type': 'string'}, + }, + }, + ) == ((('other_option',), 'value2'),) + + +def test_prepare_arguments_for_config_skips_option_missing_from_schema(): + assert module.prepare_arguments_for_config( + global_arguments=flexmock(**{'my_option.sub_option': 'value1', 'other_option': 'value2'}), + schema={ + 'type': 'object', + 'properties': { + 'my_option': {'type': 'object'}, + 'other_option': {'type': 'string'}, + }, + }, + ) == ((('other_option',), 'value2'),) + + +def test_apply_arguments_to_config_does_not_raise(): + flexmock(module).should_receive('prepare_arguments_for_config').and_return( + ( + (('foo', 'bar'), 'baz'), + (('one', 'two'), 'three'), + ) + ) + flexmock(module).should_receive('set_values') + + module.apply_arguments_to_config(config={}, schema={}, arguments={'global': flexmock()}) diff --git a/tests/unit/config/test_generate.py b/tests/unit/config/test_generate.py index 530d641c..ff71b9fb 100644 --- a/tests/unit/config/test_generate.py +++ b/tests/unit/config/test_generate.py @@ -1,70 +1,36 @@ -from collections import OrderedDict - import pytest from flexmock import flexmock from borgmatic.config import generate as module -def test_get_properties_with_simple_object(): - schema = { - 'type': 'object', - 'properties': OrderedDict( - [ - ('field1', {'example': 'Example'}), - ] - ), - } - - assert module.get_properties(schema) == schema['properties'] - - -def test_get_properties_merges_one_of_list_properties(): - schema = { - 'type': 'object', - 'oneOf': [ - { - 'properties': OrderedDict( - [ - ('field1', {'example': 'Example 1'}), - ('field2', {'example': 'Example 2'}), - ] - ), - }, - { - 'properties': OrderedDict( - [ - ('field2', {'example': 'Example 2'}), - ('field3', {'example': 'Example 3'}), - ] - ), - }, - ], - } - - assert module.get_properties(schema) == dict( - schema['oneOf'][0]['properties'], **schema['oneOf'][1]['properties'] - ) - - def test_schema_to_sample_configuration_generates_config_map_with_examples(): schema = { 'type': 'object', - 'properties': OrderedDict( + 'properties': dict( [ - ('field1', {'example': 'Example 1'}), - ('field2', {'example': 'Example 2'}), - ('field3', {'example': 'Example 3'}), + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ('field3', {'type': 'string', 'example': 'Example 3'}), ] ), } - flexmock(module).should_receive('get_properties').and_return(schema['properties']) - flexmock(module.ruamel.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').and_return(False) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'object', {'object'} + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'string', module.SCALAR_SCHEMA_TYPES, match=all + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return( + schema['properties'] + ) + flexmock(module.ruamel.yaml.comments).should_receive('CommentedMap').replace_with(dict) flexmock(module).should_receive('add_comments_to_configuration_object') config = module.schema_to_sample_configuration(schema) - assert config == OrderedDict( + assert config == dict( [ ('field1', 'Example 1'), ('field2', 'Example 2'), @@ -73,6 +39,35 @@ def test_schema_to_sample_configuration_generates_config_map_with_examples(): ) +def test_schema_to_sample_configuration_with_empty_object_generates_config_map_with_example(): + schema = { + 'type': 'object', + 'example': { + 'foo': 'Example 1', + 'baz': 'Example 2', + }, + } + flexmock(module.borgmatic.config.schema).should_receive('compare_types').and_return(False) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'object', {'object'} + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'string', module.SCALAR_SCHEMA_TYPES, match=all + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return({}) + flexmock(module.ruamel.yaml.comments).should_receive('CommentedMap').replace_with(dict) + flexmock(module).should_receive('add_comments_to_configuration_object') + + config = module.schema_to_sample_configuration(schema) + + assert config == dict( + [ + ('foo', 'Example 1'), + ('baz', 'Example 2'), + ] + ) + + def test_schema_to_sample_configuration_generates_config_sequence_of_strings_with_example(): flexmock(module.ruamel.yaml.comments).should_receive('CommentedSeq').replace_with(list) flexmock(module).should_receive('add_comments_to_configuration_sequence') @@ -88,19 +83,34 @@ def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_e 'type': 'array', 'items': { 'type': 'object', - 'properties': OrderedDict( - [('field1', {'example': 'Example 1'}), ('field2', {'example': 'Example 2'})] + 'properties': dict( + [ + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ] ), }, } - flexmock(module).should_receive('get_properties').and_return(schema['items']['properties']) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').and_return(False) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'array', {'array'} + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'object', {'object'} + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'string', module.SCALAR_SCHEMA_TYPES, match=all + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return( + schema['items']['properties'] + ) flexmock(module.ruamel.yaml.comments).should_receive('CommentedSeq').replace_with(list) flexmock(module).should_receive('add_comments_to_configuration_sequence') flexmock(module).should_receive('add_comments_to_configuration_object') config = module.schema_to_sample_configuration(schema) - assert config == [OrderedDict([('field1', 'Example 1'), ('field2', 'Example 2')])] + assert config == [dict([('field1', 'Example 1'), ('field2', 'Example 2')])] def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_multiple_types(): @@ -108,19 +118,34 @@ def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_m 'type': 'array', 'items': { 'type': ['object', 'null'], - 'properties': OrderedDict( - [('field1', {'example': 'Example 1'}), ('field2', {'example': 'Example 2'})] + 'properties': dict( + [ + ('field1', {'type': 'string', 'example': 'Example 1'}), + ('field2', {'type': 'string', 'example': 'Example 2'}), + ] ), }, } - flexmock(module).should_receive('get_properties').and_return(schema['items']['properties']) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').and_return(False) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'array', {'array'} + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + ['object', 'null'], {'object'} + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('compare_types').with_args( + 'string', module.SCALAR_SCHEMA_TYPES, match=all + ).and_return(True) + flexmock(module.borgmatic.config.schema).should_receive('get_properties').and_return( + schema['items']['properties'] + ) flexmock(module.ruamel.yaml.comments).should_receive('CommentedSeq').replace_with(list) flexmock(module).should_receive('add_comments_to_configuration_sequence') flexmock(module).should_receive('add_comments_to_configuration_object') config = module.schema_to_sample_configuration(schema) - assert config == [OrderedDict([('field1', 'Example 1'), ('field2', 'Example 2')])] + assert config == [dict([('field1', 'Example 1'), ('field2', 'Example 2')])] def test_schema_to_sample_configuration_with_unsupported_schema_raises(): @@ -133,7 +158,6 @@ def test_schema_to_sample_configuration_with_unsupported_schema_raises(): def test_merge_source_configuration_into_destination_inserts_map_fields(): destination_config = {'foo': 'dest1', 'bar': 'dest2'} source_config = {'foo': 'source1', 'baz': 'source2'} - flexmock(module).should_receive('remove_commented_out_sentinel') flexmock(module).should_receive('ruamel.yaml.comments.CommentedSeq').replace_with(list) module.merge_source_configuration_into_destination(destination_config, source_config) @@ -144,7 +168,6 @@ def test_merge_source_configuration_into_destination_inserts_map_fields(): def test_merge_source_configuration_into_destination_inserts_nested_map_fields(): destination_config = {'foo': {'first': 'dest1', 'second': 'dest2'}, 'bar': 'dest3'} source_config = {'foo': {'first': 'source1'}} - flexmock(module).should_receive('remove_commented_out_sentinel') flexmock(module).should_receive('ruamel.yaml.comments.CommentedSeq').replace_with(list) module.merge_source_configuration_into_destination(destination_config, source_config) @@ -155,7 +178,6 @@ def test_merge_source_configuration_into_destination_inserts_nested_map_fields() def test_merge_source_configuration_into_destination_inserts_sequence_fields(): destination_config = {'foo': ['dest1', 'dest2'], 'bar': ['dest3'], 'baz': ['dest4']} source_config = {'foo': ['source1'], 'bar': ['source2', 'source3']} - flexmock(module).should_receive('remove_commented_out_sentinel') flexmock(module).should_receive('ruamel.yaml.comments.CommentedSeq').replace_with(list) module.merge_source_configuration_into_destination(destination_config, source_config) @@ -170,7 +192,6 @@ def test_merge_source_configuration_into_destination_inserts_sequence_fields(): def test_merge_source_configuration_into_destination_inserts_sequence_of_maps(): destination_config = {'foo': [{'first': 'dest1', 'second': 'dest2'}], 'bar': 'dest3'} source_config = {'foo': [{'first': 'source1'}, {'other': 'source2'}]} - flexmock(module).should_receive('remove_commented_out_sentinel') flexmock(module).should_receive('ruamel.yaml.comments.CommentedSeq').replace_with(list) module.merge_source_configuration_into_destination(destination_config, source_config) diff --git a/tests/unit/config/test_normalize.py b/tests/unit/config/test_normalize.py index edef6695..29fe25e8 100644 --- a/tests/unit/config/test_normalize.py +++ b/tests/unit/config/test_normalize.py @@ -123,6 +123,114 @@ def test_normalize_sections_with_only_scalar_raises(): module.normalize_sections('test.yaml', config) +@pytest.mark.parametrize( + 'config,expected_config,produces_logs', + ( + ( + {'before_actions': ['foo', 'bar'], 'after_actions': ['baz']}, + { + 'commands': [ + {'before': 'repository', 'run': ['foo', 'bar']}, + {'after': 'repository', 'run': ['baz']}, + ] + }, + True, + ), + ( + {'before_backup': ['foo', 'bar'], 'after_backup': ['baz']}, + { + 'commands': [ + {'before': 'action', 'when': ['create'], 'run': ['foo', 'bar']}, + {'after': 'action', 'when': ['create'], 'run': ['baz']}, + ] + }, + True, + ), + ( + {'before_prune': ['foo', 'bar'], 'after_prune': ['baz']}, + { + 'commands': [ + {'before': 'action', 'when': ['prune'], 'run': ['foo', 'bar']}, + {'after': 'action', 'when': ['prune'], 'run': ['baz']}, + ] + }, + True, + ), + ( + {'before_compact': ['foo', 'bar'], 'after_compact': ['baz']}, + { + 'commands': [ + {'before': 'action', 'when': ['compact'], 'run': ['foo', 'bar']}, + {'after': 'action', 'when': ['compact'], 'run': ['baz']}, + ] + }, + True, + ), + ( + {'before_check': ['foo', 'bar'], 'after_check': ['baz']}, + { + 'commands': [ + {'before': 'action', 'when': ['check'], 'run': ['foo', 'bar']}, + {'after': 'action', 'when': ['check'], 'run': ['baz']}, + ] + }, + True, + ), + ( + {'before_extract': ['foo', 'bar'], 'after_extract': ['baz']}, + { + 'commands': [ + {'before': 'action', 'when': ['extract'], 'run': ['foo', 'bar']}, + {'after': 'action', 'when': ['extract'], 'run': ['baz']}, + ] + }, + True, + ), + ( + {'on_error': ['foo', 'bar']}, + { + 'commands': [ + { + 'after': 'error', + 'when': ['create', 'prune', 'compact', 'check'], + 'run': ['foo', 'bar'], + }, + ] + }, + True, + ), + ( + {'before_everything': ['foo', 'bar'], 'after_everything': ['baz']}, + { + 'commands': [ + {'before': 'everything', 'when': ['create'], 'run': ['foo', 'bar']}, + {'after': 'everything', 'when': ['create'], 'run': ['baz']}, + ] + }, + True, + ), + ( + {'other': 'options', 'unrelated_to': 'commands'}, + {'other': 'options', 'unrelated_to': 'commands'}, + False, + ), + ), +) +def test_normalize_commands_moves_individual_command_hooks_to_unified_commands( + config, expected_config, produces_logs +): + flexmock(module).should_receive('make_command_hook_deprecation_log').and_return(flexmock()) + + logs = module.normalize_commands('test.yaml', config) + + assert config == expected_config + + if produces_logs: + assert logs + else: + assert logs == [] + + @pytest.mark.parametrize( 'config,expected_config,produces_logs', ( @@ -251,6 +359,11 @@ def test_normalize_sections_with_only_scalar_raises(): {'repositories': [{'path': '/repo', 'label': 'foo'}]}, False, ), + ( + {'repositories': [{'path': None, 'label': 'foo'}]}, + {'repositories': []}, + False, + ), ( {'prefix': 'foo'}, {'prefix': 'foo'}, @@ -262,6 +375,7 @@ def test_normalize_applies_hard_coded_normalization_to_config( config, expected_config, produces_logs ): flexmock(module).should_receive('normalize_sections').and_return([]) + flexmock(module).should_receive('normalize_commands').and_return([]) logs = module.normalize('test.yaml', config) expected_config.setdefault('bootstrap', {}) @@ -276,6 +390,7 @@ def test_normalize_applies_hard_coded_normalization_to_config( def test_normalize_config_with_borgmatic_source_directory_warns(): flexmock(module).should_receive('normalize_sections').and_return([]) + flexmock(module).should_receive('normalize_commands').and_return([]) logs = module.normalize('test.yaml', {'borgmatic_source_directory': '~/.borgmatic'}) diff --git a/tests/unit/config/test_schema.py b/tests/unit/config/test_schema.py new file mode 100644 index 00000000..8af890fb --- /dev/null +++ b/tests/unit/config/test_schema.py @@ -0,0 +1,160 @@ +import pytest + +from borgmatic.config import schema as module + + +def test_get_properties_with_simple_object(): + schema = { + 'type': 'object', + 'properties': dict( + [ + ('field1', {'example': 'Example'}), + ] + ), + } + + assert module.get_properties(schema) == schema['properties'] + + +def test_get_properties_merges_oneof_list_properties(): + schema = { + 'type': 'object', + 'oneOf': [ + { + 'properties': dict( + [ + ('field1', {'example': 'Example 1'}), + ('field2', {'example': 'Example 2'}), + ] + ), + }, + { + 'properties': dict( + [ + ('field2', {'example': 'Example 2'}), + ('field3', {'example': 'Example 3'}), + ] + ), + }, + ], + } + + assert module.get_properties(schema) == dict( + schema['oneOf'][0]['properties'], **schema['oneOf'][1]['properties'] + ) + + +def test_get_properties_interleaves_oneof_list_properties(): + schema = { + 'type': 'object', + 'oneOf': [ + { + 'properties': dict( + [ + ('field1', {'example': 'Example 1'}), + ('field2', {'example': 'Example 2'}), + ('field3', {'example': 'Example 3'}), + ] + ), + }, + { + 'properties': dict( + [ + ('field4', {'example': 'Example 4'}), + ('field5', {'example': 'Example 5'}), + ] + ), + }, + ], + } + + assert module.get_properties(schema) == dict( + [ + ('field1', {'example': 'Example 1'}), + ('field4', {'example': 'Example 4'}), + ('field2', {'example': 'Example 2'}), + ('field5', {'example': 'Example 5'}), + ('field3', {'example': 'Example 3'}), + ] + ) + + +def test_parse_type_maps_schema_type_to_python_type(): + module.parse_type('boolean') == bool + + +def test_parse_type_with_unknown_schema_type_raises(): + with pytest.raises(ValueError): + module.parse_type('what') + + +def test_parse_type_respect_overrides_when_mapping_types(): + module.parse_type('boolean', boolean=int) == int + + +@pytest.mark.parametrize( + 'schema_type,target_types,match,expected_result', + ( + ( + 'string', + {'integer', 'string', 'boolean'}, + None, + True, + ), + ( + 'string', + {'integer', 'boolean'}, + None, + False, + ), + ( + 'string', + {'integer', 'string', 'boolean'}, + all, + True, + ), + ( + 'string', + {'integer', 'boolean'}, + all, + False, + ), + ( + ['string', 'array'], + {'integer', 'string', 'boolean'}, + None, + True, + ), + ( + ['string', 'array'], + {'integer', 'boolean'}, + None, + False, + ), + ( + ['string', 'array'], + {'integer', 'string', 'boolean', 'array'}, + all, + True, + ), + ( + ['string', 'array'], + {'integer', 'string', 'boolean'}, + all, + False, + ), + ( + ['string', 'array'], + {'integer', 'boolean'}, + all, + False, + ), + ), +) +def test_compare_types_returns_whether_schema_type_matches_target_types( + schema_type, target_types, match, expected_result +): + if match: + assert module.compare_types(schema_type, target_types, match) == expected_result + else: + assert module.compare_types(schema_type, target_types) == expected_result diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index 5bc08039..04b5bb40 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -94,13 +94,40 @@ def test_normalize_repository_path_passes_through_remote_repository(): module.normalize_repository_path(repository) == repository +def test_normalize_repository_path_passes_through_remote_repository_with_base_dir(): + repository = 'example.org:test.borg' + + flexmock(module.os.path).should_receive('abspath').never() + module.normalize_repository_path(repository, '/working') == repository + + def test_normalize_repository_path_passes_through_file_repository(): repository = 'file:///foo/bar/test.borg' - flexmock(module.os.path).should_receive('abspath').and_return('/foo/bar/test.borg') + flexmock(module.os.path).should_receive('abspath').with_args('/foo/bar/test.borg').and_return( + '/foo/bar/test.borg' + ) module.normalize_repository_path(repository) == '/foo/bar/test.borg' +def test_normalize_repository_path_passes_through_absolute_file_repository_with_base_dir(): + repository = 'file:///foo/bar/test.borg' + flexmock(module.os.path).should_receive('abspath').with_args('/foo/bar/test.borg').and_return( + '/foo/bar/test.borg' + ) + + module.normalize_repository_path(repository, '/working') == '/foo/bar/test.borg' + + +def test_normalize_repository_path_resolves_relative_file_repository_with_base_dir(): + repository = 'file://foo/bar/test.borg' + flexmock(module.os.path).should_receive('abspath').with_args( + '/working/foo/bar/test.borg' + ).and_return('/working/foo/bar/test.borg') + + module.normalize_repository_path(repository, '/working') == '/working/foo/bar/test.borg' + + def test_normalize_repository_path_passes_through_absolute_repository(): repository = '/foo/bar/test.borg' flexmock(module.os.path).should_receive('abspath').and_return(repository) @@ -108,14 +135,32 @@ def test_normalize_repository_path_passes_through_absolute_repository(): module.normalize_repository_path(repository) == repository +def test_normalize_repository_path_passes_through_absolute_repository_with_base_dir(): + repository = '/foo/bar/test.borg' + flexmock(module.os.path).should_receive('abspath').and_return(repository) + + module.normalize_repository_path(repository, '/working') == repository + + def test_normalize_repository_path_resolves_relative_repository(): repository = 'test.borg' absolute = '/foo/bar/test.borg' - flexmock(module.os.path).should_receive('abspath').and_return(absolute) + flexmock(module.os.path).should_receive('abspath').with_args(repository).and_return(absolute) module.normalize_repository_path(repository) == absolute +def test_normalize_repository_path_resolves_relative_repository_with_base_dir(): + repository = 'test.borg' + base = '/working' + absolute = '/working/test.borg' + flexmock(module.os.path).should_receive('abspath').with_args('/working/test.borg').and_return( + absolute + ) + + module.normalize_repository_path(repository, base) == absolute + + @pytest.mark.parametrize( 'first,second,expected_result', ( @@ -202,6 +247,15 @@ def test_guard_configuration_contains_repository_does_not_raise_when_repository_ ) +def test_guard_configuration_contains_repository_does_not_raise_when_repository_is_none(): + flexmock(module).should_receive('repositories_match').never() + + module.guard_configuration_contains_repository( + repository=None, + configurations={'config.yaml': {'repositories': [{'path': 'foo/bar', 'label': 'repo'}]}}, + ) + + def test_guard_configuration_contains_repository_errors_when_repository_does_not_match(): flexmock(module).should_receive('repositories_match').and_return(False) diff --git a/tests/unit/hooks/credential/test_container.py b/tests/unit/hooks/credential/test_container.py new file mode 100644 index 00000000..5f03c871 --- /dev/null +++ b/tests/unit/hooks/credential/test_container.py @@ -0,0 +1,74 @@ +import io +import sys + +import pytest +from flexmock import flexmock + +from borgmatic.hooks.credential import container as module + + +@pytest.mark.parametrize('credential_parameters', ((), ('foo', 'bar'))) +def test_load_credential_with_invalid_credential_parameters_raises(credential_parameters): + with pytest.raises(ValueError): + module.load_credential( + hook_config={}, config={}, credential_parameters=credential_parameters + ) + + +def test_load_credential_with_invalid_secret_name_raises(): + with pytest.raises(ValueError): + module.load_credential( + hook_config={}, config={}, credential_parameters=('this is invalid',) + ) + + +def test_load_credential_reads_named_secret_from_file(): + credential_stream = io.StringIO('password') + credential_stream.name = '/run/secrets/mysecret' + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('/run/secrets/mysecret').and_return(credential_stream) + + assert ( + module.load_credential(hook_config={}, config={}, credential_parameters=('mysecret',)) + == 'password' + ) + + +def test_load_credential_with_custom_secrets_directory_looks_there_for_secret_file(): + config = {'container': {'secrets_directory': '/secrets'}} + credential_stream = io.StringIO('password') + credential_stream.name = '/secrets/mysecret' + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('/secrets/mysecret').and_return(credential_stream) + + assert ( + module.load_credential( + hook_config=config['container'], config=config, credential_parameters=('mysecret',) + ) + == 'password' + ) + + +def test_load_credential_with_custom_secrets_directory_prefixes_it_with_working_directory(): + config = {'container': {'secrets_directory': 'secrets'}, 'working_directory': '/working'} + credential_stream = io.StringIO('password') + credential_stream.name = '/working/secrets/mysecret' + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('/working/secrets/mysecret').and_return( + credential_stream + ) + + assert ( + module.load_credential( + hook_config=config['container'], config=config, credential_parameters=('mysecret',) + ) + == 'password' + ) + + +def test_load_credential_with_file_not_found_error_raises(): + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('/run/secrets/mysecret').and_raise(FileNotFoundError) + + with pytest.raises(ValueError): + module.load_credential(hook_config={}, config={}, credential_parameters=('mysecret',)) diff --git a/tests/unit/hooks/credential/test_file.py b/tests/unit/hooks/credential/test_file.py new file mode 100644 index 00000000..42827a85 --- /dev/null +++ b/tests/unit/hooks/credential/test_file.py @@ -0,0 +1,96 @@ +import io +import sys + +import pytest +from flexmock import flexmock + +from borgmatic.hooks.credential import file as module + + +@pytest.mark.parametrize('credential_parameters', ((), ('foo', 'bar'))) +def test_load_credential_with_invalid_credential_parameters_raises(credential_parameters): + with pytest.raises(ValueError): + module.load_credential( + hook_config={}, config={}, credential_parameters=credential_parameters + ) + + +def test_load_credential_with_invalid_credential_name_raises(): + with pytest.raises(ValueError): + module.load_credential( + hook_config={}, config={}, credential_parameters=('this is invalid',) + ) + + +def test_load_credential_reads_named_credential_from_file(): + credential_stream = io.StringIO('password') + credential_stream.name = '/credentials/mycredential' + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os.path).should_receive('expanduser').with_args( + '/credentials/mycredential' + ).and_return('/credentials/mycredential') + builtins.should_receive('open').with_args('/credentials/mycredential').and_return( + credential_stream + ) + + assert ( + module.load_credential( + hook_config={}, config={}, credential_parameters=('/credentials/mycredential',) + ) + == 'password' + ) + + +def test_load_credential_reads_named_credential_from_file_using_working_directory(): + credential_stream = io.StringIO('password') + credential_stream.name = '/working/credentials/mycredential' + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os.path).should_receive('expanduser').with_args( + 'credentials/mycredential' + ).and_return('credentials/mycredential') + builtins.should_receive('open').with_args('/working/credentials/mycredential').and_return( + credential_stream + ) + + assert ( + module.load_credential( + hook_config={}, + config={'working_directory': '/working'}, + credential_parameters=('credentials/mycredential',), + ) + == 'password' + ) + + +def test_load_credential_with_file_not_found_error_raises(): + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os.path).should_receive('expanduser').with_args( + '/credentials/mycredential' + ).and_return('/credentials/mycredential') + builtins.should_receive('open').with_args('/credentials/mycredential').and_raise( + FileNotFoundError + ) + + with pytest.raises(ValueError): + module.load_credential( + hook_config={}, config={}, credential_parameters=('/credentials/mycredential',) + ) + + +def test_load_credential_reads_named_credential_from_expanded_directory(): + credential_stream = io.StringIO('password') + credential_stream.name = '/root/credentials/mycredential' + builtins = flexmock(sys.modules['builtins']) + flexmock(module.os.path).should_receive('expanduser').with_args( + '~/credentials/mycredential' + ).and_return('/root/credentials/mycredential') + builtins.should_receive('open').with_args('/root/credentials/mycredential').and_return( + credential_stream + ) + + assert ( + module.load_credential( + hook_config={}, config={}, credential_parameters=('~/credentials/mycredential',) + ) + == 'password' + ) diff --git a/tests/unit/hooks/credential/test_keepassxc.py b/tests/unit/hooks/credential/test_keepassxc.py new file mode 100644 index 00000000..ccb9173b --- /dev/null +++ b/tests/unit/hooks/credential/test_keepassxc.py @@ -0,0 +1,219 @@ +import pytest +from flexmock import flexmock + +from borgmatic.hooks.credential import keepassxc as module + + +@pytest.mark.parametrize('credential_parameters', ((), ('foo',), ('foo', 'bar', 'baz'))) +def test_load_credential_with_invalid_credential_parameters_raises(credential_parameters): + flexmock(module.borgmatic.execute).should_receive('execute_command_and_capture_output').never() + + with pytest.raises(ValueError): + module.load_credential( + hook_config={}, config={}, credential_parameters=credential_parameters + ) + + +def test_load_credential_with_missing_database_raises(): + flexmock(module.os.path).should_receive('expanduser').with_args('database.kdbx').and_return( + 'database.kdbx' + ) + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.execute).should_receive('execute_command_and_capture_output').never() + + with pytest.raises(ValueError): + module.load_credential( + hook_config={}, config={}, credential_parameters=('database.kdbx', 'mypassword') + ) + + +def test_load_credential_with_present_database_fetches_password_from_keepassxc(): + flexmock(module.os.path).should_receive('expanduser').with_args('database.kdbx').and_return( + 'database.kdbx' + ) + flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).with_args( + ( + 'keepassxc-cli', + 'show', + '--show-protected', + '--attributes', + 'Password', + 'database.kdbx', + 'mypassword', + ) + ).and_return( + 'password' + ).once() + + assert ( + module.load_credential( + hook_config={}, config={}, credential_parameters=('database.kdbx', 'mypassword') + ) + == 'password' + ) + + +def test_load_credential_with_custom_keepassxc_cli_command_calls_it(): + flexmock(module.os.path).should_receive('expanduser').with_args('database.kdbx').and_return( + 'database.kdbx' + ) + config = {'keepassxc': {'keepassxc_cli_command': '/usr/local/bin/keepassxc-cli --some-option'}} + flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).with_args( + ( + '/usr/local/bin/keepassxc-cli', + '--some-option', + 'show', + '--show-protected', + '--attributes', + 'Password', + 'database.kdbx', + 'mypassword', + ) + ).and_return( + 'password' + ).once() + + assert ( + module.load_credential( + hook_config=config['keepassxc'], + config=config, + credential_parameters=('database.kdbx', 'mypassword'), + ) + == 'password' + ) + + +def test_load_credential_with_expanded_directory_with_present_database_fetches_password_from_keepassxc(): + flexmock(module.os.path).should_receive('expanduser').with_args('~/database.kdbx').and_return( + '/root/database.kdbx' + ) + flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).with_args( + ( + 'keepassxc-cli', + 'show', + '--show-protected', + '--attributes', + 'Password', + '/root/database.kdbx', + 'mypassword', + ) + ).and_return( + 'password' + ).once() + + assert ( + module.load_credential( + hook_config={}, config={}, credential_parameters=('~/database.kdbx', 'mypassword') + ) + == 'password' + ) + + +def test_load_credential_with_key_file(): + flexmock(module.os.path).should_receive('expanduser').with_args('database.kdbx').and_return( + 'database.kdbx' + ) + flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).with_args( + ( + 'keepassxc-cli', + 'show', + '--show-protected', + '--attributes', + 'Password', + '--key-file', + '/path/to/keyfile', + 'database.kdbx', + 'mypassword', + ) + ).and_return( + 'password' + ).once() + + assert ( + module.load_credential( + hook_config={'key_file': '/path/to/keyfile'}, + config={}, + credential_parameters=('database.kdbx', 'mypassword'), + ) + == 'password' + ) + + +def test_load_credential_with_yubikey(): + flexmock(module.os.path).should_receive('expanduser').with_args('database.kdbx').and_return( + 'database.kdbx' + ) + flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).with_args( + ( + 'keepassxc-cli', + 'show', + '--show-protected', + '--attributes', + 'Password', + '--yubikey', + '1:7370001', + 'database.kdbx', + 'mypassword', + ) + ).and_return( + 'password' + ).once() + + assert ( + module.load_credential( + hook_config={'yubikey': '1:7370001'}, + config={}, + credential_parameters=('database.kdbx', 'mypassword'), + ) + == 'password' + ) + + +def test_load_credential_with_key_file_and_yubikey(): + flexmock(module.os.path).should_receive('expanduser').with_args('database.kdbx').and_return( + 'database.kdbx' + ) + flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).with_args( + ( + 'keepassxc-cli', + 'show', + '--show-protected', + '--attributes', + 'Password', + '--key-file', + '/path/to/keyfile', + '--yubikey', + '2', + 'database.kdbx', + 'mypassword', + ) + ).and_return( + 'password' + ).once() + + assert ( + module.load_credential( + hook_config={'key_file': '/path/to/keyfile', 'yubikey': '2'}, + config={}, + credential_parameters=('database.kdbx', 'mypassword'), + ) + == 'password' + ) diff --git a/tests/unit/hooks/credential/test_parse.py b/tests/unit/hooks/credential/test_parse.py new file mode 100644 index 00000000..c1c9e1b4 --- /dev/null +++ b/tests/unit/hooks/credential/test_parse.py @@ -0,0 +1,95 @@ +import pytest +from flexmock import flexmock + +from borgmatic.hooks.credential import parse as module + + +def test_hash_adapter_is_always_equal(): + assert module.Hash_adapter({1: 2}) == module.Hash_adapter({3: 4}) + + +def test_hash_adapter_alwaysh_hashes_the_same(): + assert hash(module.Hash_adapter({1: 2})) == hash(module.Hash_adapter({3: 4})) + + +def test_cache_ignoring_unhashable_arguments_caches_arguments_after_first_call(): + hashable = 3 + unhashable = {1, 2} + calls = 0 + + @module.cache_ignoring_unhashable_arguments + def function(first, second, third): + nonlocal calls + calls += 1 + + assert first == hashable + assert second == unhashable + assert third == unhashable + + return first + + assert function(hashable, unhashable, third=unhashable) == hashable + assert calls == 1 + + assert function(hashable, unhashable, third=unhashable) == hashable + assert calls == 1 + + +def test_resolve_credential_passes_through_string_without_credential(): + module.resolve_credential.cache_clear() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').never() + + assert module.resolve_credential('{no credentials here}', config={}) == '{no credentials here}' + + +def test_resolve_credential_passes_through_none(): + module.resolve_credential.cache_clear() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').never() + + assert module.resolve_credential(None, config={}) is None + + +@pytest.mark.parametrize('invalid_value', ('{credential}', '{credential }', '{credential systemd}')) +def test_resolve_credential_with_invalid_credential_raises(invalid_value): + module.resolve_credential.cache_clear() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').never() + + with pytest.raises(ValueError): + module.resolve_credential(invalid_value, config={}) + + +def test_resolve_credential_with_valid_credential_loads_credential(): + module.resolve_credential.cache_clear() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( + 'load_credential', + {}, + 'systemd', + ('mycredential',), + ).and_return('result').once() + + assert module.resolve_credential('{credential systemd mycredential}', config={}) == 'result' + + +def test_resolve_credential_with_valid_credential_and_quoted_parameters_loads_credential(): + module.resolve_credential.cache_clear() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( + 'load_credential', + {}, + 'systemd', + ('my credential',), + ).and_return('result').once() + + assert module.resolve_credential('{credential systemd "my credential"}', config={}) == 'result' + + +def test_resolve_credential_caches_credential_after_first_call(): + module.resolve_credential.cache_clear() + flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( + 'load_credential', + {}, + 'systemd', + ('mycredential',), + ).and_return('result').once() + + assert module.resolve_credential('{credential systemd mycredential}', config={}) == 'result' + assert module.resolve_credential('{credential systemd mycredential}', config={}) == 'result' diff --git a/tests/unit/hooks/credential/test_systemd.py b/tests/unit/hooks/credential/test_systemd.py new file mode 100644 index 00000000..32133883 --- /dev/null +++ b/tests/unit/hooks/credential/test_systemd.py @@ -0,0 +1,63 @@ +import io +import sys + +import pytest +from flexmock import flexmock + +from borgmatic.hooks.credential import systemd as module + + +@pytest.mark.parametrize('credential_parameters', ((), ('foo', 'bar'))) +def test_load_credential_with_invalid_credential_parameters_raises(credential_parameters): + flexmock(module.os.environ).should_receive('get').never() + + with pytest.raises(ValueError): + module.load_credential( + hook_config={}, config={}, credential_parameters=credential_parameters + ) + + +def test_load_credential_without_credentials_directory_raises(): + flexmock(module.os.environ).should_receive('get').with_args('CREDENTIALS_DIRECTORY').and_return( + None + ) + + with pytest.raises(ValueError): + module.load_credential(hook_config={}, config={}, credential_parameters=('mycredential',)) + + +def test_load_credential_with_invalid_credential_name_raises(): + flexmock(module.os.environ).should_receive('get').with_args('CREDENTIALS_DIRECTORY').and_return( + '/var' + ) + + with pytest.raises(ValueError): + module.load_credential( + hook_config={}, config={}, credential_parameters=('../../my!@#$credential',) + ) + + +def test_load_credential_reads_named_credential_from_file(): + flexmock(module.os.environ).should_receive('get').with_args('CREDENTIALS_DIRECTORY').and_return( + '/var' + ) + credential_stream = io.StringIO('password') + credential_stream.name = '/var/borgmatic.pw' + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('/var/borgmatic.pw').and_return(credential_stream) + + assert ( + module.load_credential(hook_config={}, config={}, credential_parameters=('borgmatic.pw',)) + == 'password' + ) + + +def test_load_credential_with_file_not_found_error_raises(): + flexmock(module.os.environ).should_receive('get').with_args('CREDENTIALS_DIRECTORY').and_return( + '/var' + ) + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('/var/mycredential').and_raise(FileNotFoundError) + + with pytest.raises(ValueError): + module.load_credential(hook_config={}, config={}, credential_parameters=('mycredential',)) diff --git a/tests/unit/hooks/data_source/test_btrfs.py b/tests/unit/hooks/data_source/test_btrfs.py index c69d5ac1..fb44b91d 100644 --- a/tests/unit/hooks/data_source/test_btrfs.py +++ b/tests/unit/hooks/data_source/test_btrfs.py @@ -1,7 +1,7 @@ import pytest from flexmock import flexmock -from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type +from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type from borgmatic.hooks.data_source import btrfs as module @@ -49,12 +49,70 @@ def test_get_subvolume_mount_points_with_findmnt_json_missing_filesystems_errors module.get_subvolume_mount_points('findmnt') +def test_get_subvolume_property_with_invalid_btrfs_output_errors(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return('invalid') + + with pytest.raises(ValueError): + module.get_subvolume_property('btrfs', '/foo', 'ro') + + +def test_get_subvolume_property_with_true_output_returns_true_bool(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return('ro=true') + + assert module.get_subvolume_property('btrfs', '/foo', 'ro') is True + + +def test_get_subvolume_property_with_false_output_returns_false_bool(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return('ro=false') + + assert module.get_subvolume_property('btrfs', '/foo', 'ro') is False + + +def test_get_subvolume_property_passes_through_general_value(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return('thing=value') + + assert module.get_subvolume_property('btrfs', '/foo', 'thing') == 'value' + + +def test_omit_read_only_subvolume_mount_points_filters_out_read_only(): + flexmock(module).should_receive('get_subvolume_property').with_args( + 'btrfs', '/foo', 'ro' + ).and_return(False) + flexmock(module).should_receive('get_subvolume_property').with_args( + 'btrfs', '/bar', 'ro' + ).and_return(True) + flexmock(module).should_receive('get_subvolume_property').with_args( + 'btrfs', '/baz', 'ro' + ).and_return(False) + + assert module.omit_read_only_subvolume_mount_points('btrfs', ('/foo', '/bar', '/baz')) == ( + '/foo', + '/baz', + ) + + def test_get_subvolumes_collects_subvolumes_matching_patterns(): flexmock(module).should_receive('get_subvolume_mount_points').and_return(('/mnt1', '/mnt2')) + flexmock(module).should_receive('omit_read_only_subvolume_mount_points').and_return( + ('/mnt1', '/mnt2') + ) + contained_pattern = Pattern( + '/mnt1', + type=Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' - ).with_args('/mnt1', object).and_return((Pattern('/mnt1'),)) + ).with_args('/mnt1', object).and_return((contained_pattern,)) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' ).with_args('/mnt2', object).and_return(()) @@ -66,11 +124,82 @@ def test_get_subvolumes_collects_subvolumes_matching_patterns(): Pattern('/mnt1'), Pattern('/mnt3'), ], - ) == (module.Subvolume('/mnt1', contained_patterns=(Pattern('/mnt1'),)),) + ) == (module.Subvolume('/mnt1', contained_patterns=(contained_pattern,)),) + + +def test_get_subvolumes_skips_non_root_patterns(): + flexmock(module).should_receive('get_subvolume_mount_points').and_return(('/mnt1', '/mnt2')) + flexmock(module).should_receive('omit_read_only_subvolume_mount_points').and_return( + ('/mnt1', '/mnt2') + ) + + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/mnt1', object).and_return( + ( + Pattern( + '/mnt1', + type=Pattern_type.EXCLUDE, + source=Pattern_source.CONFIG, + ), + ) + ) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/mnt2', object).and_return(()) + + assert ( + module.get_subvolumes( + 'btrfs', + 'findmnt', + patterns=[ + Pattern('/mnt1'), + Pattern('/mnt3'), + ], + ) + == () + ) + + +def test_get_subvolumes_skips_non_config_patterns(): + flexmock(module).should_receive('get_subvolume_mount_points').and_return(('/mnt1', '/mnt2')) + flexmock(module).should_receive('omit_read_only_subvolume_mount_points').and_return( + ('/mnt1', '/mnt2') + ) + + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/mnt1', object).and_return( + ( + Pattern( + '/mnt1', + type=Pattern_type.ROOT, + source=Pattern_source.HOOK, + ), + ) + ) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/mnt2', object).and_return(()) + + assert ( + module.get_subvolumes( + 'btrfs', + 'findmnt', + patterns=[ + Pattern('/mnt1'), + Pattern('/mnt3'), + ], + ) + == () + ) def test_get_subvolumes_without_patterns_collects_all_subvolumes(): flexmock(module).should_receive('get_subvolume_mount_points').and_return(('/mnt1', '/mnt2')) + flexmock(module).should_receive('omit_read_only_subvolume_mount_points').and_return( + ('/mnt1', '/mnt2') + ) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' @@ -520,6 +649,18 @@ def test_remove_data_source_dumps_deletes_snapshots(): flexmock(module).should_receive('delete_snapshot').with_args( 'btrfs', '/mnt/subvol2/.borgmatic-5678/mnt/subvol2' ).never() + flexmock(module.os.path).should_receive('isdir').with_args( + '/mnt/subvol1/.borgmatic-1234' + ).and_return(True) + flexmock(module.os.path).should_receive('isdir').with_args( + '/mnt/subvol1/.borgmatic-5678' + ).and_return(True) + flexmock(module.os.path).should_receive('isdir').with_args( + '/mnt/subvol2/.borgmatic-1234' + ).and_return(True) + flexmock(module.os.path).should_receive('isdir').with_args( + '/mnt/subvol2/.borgmatic-5678' + ).and_return(True) flexmock(module.shutil).should_receive('rmtree').with_args( '/mnt/subvol1/.borgmatic-1234' ).once() @@ -846,3 +987,48 @@ def test_remove_data_source_dumps_with_delete_snapshot_called_process_error_bail borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) + + +def test_remove_data_source_dumps_with_root_subvolume_skips_duplicate_removal(): + config = {'btrfs': {}} + flexmock(module).should_receive('get_subvolumes').and_return( + (module.Subvolume('/', contained_patterns=(Pattern('/etc'),)),) + ) + + flexmock(module).should_receive('make_snapshot_path').with_args('/').and_return( + '/.borgmatic-1234' + ) + + flexmock(module.borgmatic.config.paths).should_receive( + 'replace_temporary_subdirectory_with_glob' + ).with_args( + '/.borgmatic-1234', + temporary_directory_prefix=module.BORGMATIC_SNAPSHOT_PREFIX, + ).and_return( + '/.borgmatic-*' + ) + + flexmock(module.glob).should_receive('glob').with_args('/.borgmatic-*').and_return( + ('/.borgmatic-1234', '/.borgmatic-5678') + ) + + flexmock(module.os.path).should_receive('isdir').with_args('/.borgmatic-1234').and_return( + True + ).and_return(False) + flexmock(module.os.path).should_receive('isdir').with_args('/.borgmatic-5678').and_return( + True + ).and_return(False) + + flexmock(module).should_receive('delete_snapshot').with_args('btrfs', '/.borgmatic-1234').once() + flexmock(module).should_receive('delete_snapshot').with_args('btrfs', '/.borgmatic-5678').once() + + flexmock(module.os.path).should_receive('isdir').with_args('').and_return(False) + + flexmock(module.shutil).should_receive('rmtree').never() + + module.remove_data_source_dumps( + hook_config=config['btrfs'], + config=config, + borgmatic_runtime_directory='/run/borgmatic', + dry_run=False, + ) diff --git a/tests/unit/hooks/data_source/test_lvm.py b/tests/unit/hooks/data_source/test_lvm.py index 49159905..34cf5250 100644 --- a/tests/unit/hooks/data_source/test_lvm.py +++ b/tests/unit/hooks/data_source/test_lvm.py @@ -1,7 +1,7 @@ import pytest from flexmock import flexmock -from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type +from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type from borgmatic.hooks.data_source import lvm as module @@ -37,14 +37,20 @@ def test_get_logical_volumes_filters_by_patterns(): } ''' ) - contained = {Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir')} + contained = { + Pattern('/mnt/lvolume', source=Pattern_source.CONFIG), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.CONFIG), + } flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' ).with_args(None, contained).never() flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' ).with_args('/mnt/lvolume', contained).and_return( - (Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir')) + ( + Pattern('/mnt/lvolume', source=Pattern_source.CONFIG), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.CONFIG), + ) ) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' @@ -54,17 +60,116 @@ def test_get_logical_volumes_filters_by_patterns(): ).with_args('/mnt/notlvm', contained).never() assert module.get_logical_volumes( - 'lsblk', patterns=(Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir')) + 'lsblk', + patterns=( + Pattern('/mnt/lvolume', source=Pattern_source.CONFIG), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.CONFIG), + ), ) == ( module.Logical_volume( name='vgroup-lvolume', device_path='/dev/mapper/vgroup-lvolume', mount_point='/mnt/lvolume', - contained_patterns=(Pattern('/mnt/lvolume'), Pattern('/mnt/lvolume/subdir')), + contained_patterns=( + Pattern('/mnt/lvolume', source=Pattern_source.CONFIG), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.CONFIG), + ), ), ) +def test_get_logical_volumes_skips_non_root_patterns(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( + ''' + { + "blockdevices": [ + { + "name": "vgroup-lvolume", + "path": "/dev/mapper/vgroup-lvolume", + "mountpoint": "/mnt/lvolume", + "type": "lvm" + } + ] + } + ''' + ) + contained = { + Pattern('/mnt/lvolume', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG), + Pattern('/mnt/lvolume/subdir', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG), + } + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args(None, contained).never() + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/mnt/lvolume', contained).and_return( + ( + Pattern('/mnt/lvolume', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG), + Pattern('/mnt/lvolume/subdir', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG), + ) + ) + + assert ( + module.get_logical_volumes( + 'lsblk', + patterns=( + Pattern('/mnt/lvolume', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG), + Pattern( + '/mnt/lvolume/subdir', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG + ), + ), + ) + == () + ) + + +def test_get_logical_volumes_skips_non_config_patterns(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( + ''' + { + "blockdevices": [ + { + "name": "vgroup-lvolume", + "path": "/dev/mapper/vgroup-lvolume", + "mountpoint": "/mnt/lvolume", + "type": "lvm" + } + ] + } + ''' + ) + contained = { + Pattern('/mnt/lvolume', source=Pattern_source.HOOK), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.HOOK), + } + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args(None, contained).never() + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/mnt/lvolume', contained).and_return( + ( + Pattern('/mnt/lvolume', source=Pattern_source.HOOK), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.HOOK), + ) + ) + + assert ( + module.get_logical_volumes( + 'lsblk', + patterns=( + Pattern('/mnt/lvolume', source=Pattern_source.HOOK), + Pattern('/mnt/lvolume/subdir', source=Pattern_source.HOOK), + ), + ) + == () + ) + + def test_get_logical_volumes_with_invalid_lsblk_json_errors(): flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' @@ -138,13 +243,13 @@ def test_snapshot_logical_volume_with_non_percentage_snapshot_name_uses_lvcreate ( ( Pattern('/foo/bar/baz'), - Pattern('/run/borgmatic/lvm_snapshots/./foo/bar/baz'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./foo/bar/baz'), ), - (Pattern('/foo/bar'), Pattern('/run/borgmatic/lvm_snapshots/./foo/bar')), + (Pattern('/foo/bar'), Pattern('/run/borgmatic/lvm_snapshots/b33f/./foo/bar')), ( Pattern('^/foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION), Pattern( - '^/run/borgmatic/lvm_snapshots/./foo/bar', + '^/run/borgmatic/lvm_snapshots/b33f/./foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION, ), @@ -152,40 +257,48 @@ def test_snapshot_logical_volume_with_non_percentage_snapshot_name_uses_lvcreate ( Pattern('/foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION), Pattern( - '/run/borgmatic/lvm_snapshots/./foo/bar', + '/run/borgmatic/lvm_snapshots/b33f/./foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION, ), ), - (Pattern('/foo'), Pattern('/run/borgmatic/lvm_snapshots/./foo')), - (Pattern('/'), Pattern('/run/borgmatic/lvm_snapshots/./')), + (Pattern('/foo'), Pattern('/run/borgmatic/lvm_snapshots/b33f/./foo')), + (Pattern('/'), Pattern('/run/borgmatic/lvm_snapshots/b33f/./')), ), ) def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path( pattern, expected_pattern ): - assert module.make_borg_snapshot_pattern(pattern, '/run/borgmatic') == expected_pattern + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) + + assert ( + module.make_borg_snapshot_pattern( + pattern, flexmock(mount_point='/something'), '/run/borgmatic' + ) + == expected_pattern + ) def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns(): config = {'lvm': {}} patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')] - flexmock(module).should_receive('get_logical_volumes').and_return( - ( - module.Logical_volume( - name='lvolume1', - device_path='/dev/lvolume1', - mount_point='/mnt/lvolume1', - contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), - ), - module.Logical_volume( - name='lvolume2', - device_path='/dev/lvolume2', - mount_point='/mnt/lvolume2', - contained_patterns=(Pattern('/mnt/lvolume2'),), - ), - ) + logical_volumes = ( + module.Logical_volume( + name='lvolume1', + device_path='/dev/lvolume1', + mount_point='/mnt/lvolume1', + contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), + ), + module.Logical_volume( + name='lvolume2', + device_path='/dev/lvolume2', + mount_point='/mnt/lvolume2', + contained_patterns=(Pattern('/mnt/lvolume2'),), + ), ) + flexmock(module).should_receive('get_logical_volumes').and_return(logical_volumes) flexmock(module.os).should_receive('getpid').and_return(1234) flexmock(module).should_receive('snapshot_logical_volume').with_args( 'lvcreate', 'lvolume1_borgmatic-1234', '/dev/lvolume1', module.DEFAULT_SNAPSHOT_SIZE @@ -203,18 +316,21 @@ def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns(): ).and_return( (module.Snapshot(name='lvolume2_borgmatic-1234', device_path='/dev/lvolume2_snap'),) ) + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) flexmock(module).should_receive('mount_snapshot').with_args( - 'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume1' + 'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' ).once() flexmock(module).should_receive('mount_snapshot').with_args( - 'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume2' + 'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' ).once() flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume1/subdir'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir')) + Pattern('/mnt/lvolume1/subdir'), logical_volumes[0], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir')) flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume2'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2')) + Pattern('/mnt/lvolume2'), logical_volumes[1], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2')) assert ( module.dump_data_sources( @@ -229,8 +345,8 @@ def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns(): ) assert patterns == [ - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'), - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2'), ] @@ -259,22 +375,21 @@ def test_dump_data_sources_with_no_logical_volumes_skips_snapshots(): def test_dump_data_sources_uses_snapshot_size_for_snapshot(): config = {'lvm': {'snapshot_size': '1000PB'}} patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')] - flexmock(module).should_receive('get_logical_volumes').and_return( - ( - module.Logical_volume( - name='lvolume1', - device_path='/dev/lvolume1', - mount_point='/mnt/lvolume1', - contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), - ), - module.Logical_volume( - name='lvolume2', - device_path='/dev/lvolume2', - mount_point='/mnt/lvolume2', - contained_patterns=(Pattern('/mnt/lvolume2'),), - ), - ) + logical_volumes = ( + module.Logical_volume( + name='lvolume1', + device_path='/dev/lvolume1', + mount_point='/mnt/lvolume1', + contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), + ), + module.Logical_volume( + name='lvolume2', + device_path='/dev/lvolume2', + mount_point='/mnt/lvolume2', + contained_patterns=(Pattern('/mnt/lvolume2'),), + ), ) + flexmock(module).should_receive('get_logical_volumes').and_return(logical_volumes) flexmock(module.os).should_receive('getpid').and_return(1234) flexmock(module).should_receive('snapshot_logical_volume').with_args( 'lvcreate', @@ -298,18 +413,21 @@ def test_dump_data_sources_uses_snapshot_size_for_snapshot(): ).and_return( (module.Snapshot(name='lvolume2_borgmatic-1234', device_path='/dev/lvolume2_snap'),) ) + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) flexmock(module).should_receive('mount_snapshot').with_args( - 'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume1' + 'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' ).once() flexmock(module).should_receive('mount_snapshot').with_args( - 'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume2' + 'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' ).once() flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume1/subdir'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir')) + Pattern('/mnt/lvolume1/subdir'), logical_volumes[0], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir')) flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume2'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2')) + Pattern('/mnt/lvolume2'), logical_volumes[1], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2')) assert ( module.dump_data_sources( @@ -324,8 +442,8 @@ def test_dump_data_sources_uses_snapshot_size_for_snapshot(): ) assert patterns == [ - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'), - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2'), ] @@ -339,22 +457,21 @@ def test_dump_data_sources_uses_custom_commands(): }, } patterns = [Pattern('/mnt/lvolume1/subdir'), Pattern('/mnt/lvolume2')] - flexmock(module).should_receive('get_logical_volumes').and_return( - ( - module.Logical_volume( - name='lvolume1', - device_path='/dev/lvolume1', - mount_point='/mnt/lvolume1', - contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), - ), - module.Logical_volume( - name='lvolume2', - device_path='/dev/lvolume2', - mount_point='/mnt/lvolume2', - contained_patterns=(Pattern('/mnt/lvolume2'),), - ), - ) + logical_volumes = ( + module.Logical_volume( + name='lvolume1', + device_path='/dev/lvolume1', + mount_point='/mnt/lvolume1', + contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), + ), + module.Logical_volume( + name='lvolume2', + device_path='/dev/lvolume2', + mount_point='/mnt/lvolume2', + contained_patterns=(Pattern('/mnt/lvolume2'),), + ), ) + flexmock(module).should_receive('get_logical_volumes').and_return(logical_volumes) flexmock(module.os).should_receive('getpid').and_return(1234) flexmock(module).should_receive('snapshot_logical_volume').with_args( '/usr/local/bin/lvcreate', @@ -378,18 +495,25 @@ def test_dump_data_sources_uses_custom_commands(): ).and_return( (module.Snapshot(name='lvolume2_borgmatic-1234', device_path='/dev/lvolume2_snap'),) ) + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) flexmock(module).should_receive('mount_snapshot').with_args( - '/usr/local/bin/mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume1' + '/usr/local/bin/mount', + '/dev/lvolume1_snap', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).once() flexmock(module).should_receive('mount_snapshot').with_args( - '/usr/local/bin/mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume2' + '/usr/local/bin/mount', + '/dev/lvolume2_snap', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).once() flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume1/subdir'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir')) + Pattern('/mnt/lvolume1/subdir'), logical_volumes[0], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir')) flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume2'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2')) + Pattern('/mnt/lvolume2'), logical_volumes[1], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2')) assert ( module.dump_data_sources( @@ -404,8 +528,8 @@ def test_dump_data_sources_uses_custom_commands(): ) assert patterns == [ - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'), - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2'), ] @@ -463,22 +587,21 @@ def test_dump_data_sources_with_dry_run_skips_snapshots_and_does_not_touch_patte def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained_patterns(): config = {'lvm': {}} patterns = [Pattern('/hmm')] - flexmock(module).should_receive('get_logical_volumes').and_return( - ( - module.Logical_volume( - name='lvolume1', - device_path='/dev/lvolume1', - mount_point='/mnt/lvolume1', - contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), - ), - module.Logical_volume( - name='lvolume2', - device_path='/dev/lvolume2', - mount_point='/mnt/lvolume2', - contained_patterns=(Pattern('/mnt/lvolume2'),), - ), - ) + logical_volumes = ( + module.Logical_volume( + name='lvolume1', + device_path='/dev/lvolume1', + mount_point='/mnt/lvolume1', + contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), + ), + module.Logical_volume( + name='lvolume2', + device_path='/dev/lvolume2', + mount_point='/mnt/lvolume2', + contained_patterns=(Pattern('/mnt/lvolume2'),), + ), ) + flexmock(module).should_receive('get_logical_volumes').and_return(logical_volumes) flexmock(module.os).should_receive('getpid').and_return(1234) flexmock(module).should_receive('snapshot_logical_volume').with_args( 'lvcreate', 'lvolume1_borgmatic-1234', '/dev/lvolume1', module.DEFAULT_SNAPSHOT_SIZE @@ -496,18 +619,21 @@ def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained ).and_return( (module.Snapshot(name='lvolume2_borgmatic-1234', device_path='/dev/lvolume2_snap'),) ) + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) flexmock(module).should_receive('mount_snapshot').with_args( - 'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume1' + 'mount', '/dev/lvolume1_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' ).once() flexmock(module).should_receive('mount_snapshot').with_args( - 'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/mnt/lvolume2' + 'mount', '/dev/lvolume2_snap', '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' ).once() flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume1/subdir'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir')) + Pattern('/mnt/lvolume1/subdir'), logical_volumes[0], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir')) flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/lvolume2'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2')) + Pattern('/mnt/lvolume2'), logical_volumes[1], '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2')) assert ( module.dump_data_sources( @@ -523,8 +649,8 @@ def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained assert patterns == [ Pattern('/hmm'), - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume1/subdir'), - Pattern('/run/borgmatic/lvm_snapshots/./mnt/lvolume2'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume1/subdir'), + Pattern('/run/borgmatic/lvm_snapshots/b33f/./mnt/lvolume2'), ] @@ -694,16 +820,19 @@ def test_remove_data_source_dumps_unmounts_and_remove_snapshots(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).once() flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).once() flexmock(module).should_receive('get_snapshots').and_return( ( @@ -799,9 +928,11 @@ def test_remove_data_source_dumps_with_missing_snapshot_directory_skips_unmount( flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots' + '/run/borgmatic/lvm_snapshots/b33f' ).and_return(False) flexmock(module.shutil).should_receive('rmtree').never() flexmock(module).should_receive('unmount_snapshot').never() @@ -843,24 +974,92 @@ def test_remove_data_source_dumps_with_missing_snapshot_mount_path_skips_unmount flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots' + '/run/borgmatic/lvm_snapshots/b33f' ).and_return(True) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots/mnt/lvolume1' + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' ).and_return(False) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots/mnt/lvolume2' + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' ).and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).never() flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', + ).once() + flexmock(module).should_receive('get_snapshots').and_return( + ( + module.Snapshot('lvolume1_borgmatic-1234', '/dev/lvolume1'), + module.Snapshot('lvolume2_borgmatic-1234', '/dev/lvolume2'), + ), + ) + flexmock(module).should_receive('remove_snapshot').with_args('lvremove', '/dev/lvolume1').once() + flexmock(module).should_receive('remove_snapshot').with_args('lvremove', '/dev/lvolume2').once() + + module.remove_data_source_dumps( + hook_config=config['lvm'], + config=config, + borgmatic_runtime_directory='/run/borgmatic', + dry_run=False, + ) + + +def test_remove_data_source_dumps_with_empty_snapshot_mount_path_skips_unmount(): + config = {'lvm': {}} + flexmock(module).should_receive('get_logical_volumes').and_return( + ( + module.Logical_volume( + name='lvolume1', + device_path='/dev/lvolume1', + mount_point='/mnt/lvolume1', + contained_patterns=(Pattern('/mnt/lvolume1/subdir'),), + ), + module.Logical_volume( + name='lvolume2', + device_path='/dev/lvolume2', + mount_point='/mnt/lvolume2', + contained_patterns=(Pattern('/mnt/lvolume2'),), + ), + ) + ) + flexmock(module.borgmatic.config.paths).should_receive( + 'replace_temporary_subdirectory_with_glob' + ).and_return('/run/borgmatic') + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) + flexmock(module.os.path).should_receive('isdir').with_args( + '/run/borgmatic/lvm_snapshots/b33f' + ).and_return(True) + flexmock(module.os.path).should_receive('isdir').with_args( + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' + ).and_return(True) + flexmock(module.os).should_receive('listdir').with_args( + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' + ).and_return([]) + flexmock(module.os.path).should_receive('isdir').with_args( + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' + ).and_return(True) + flexmock(module.os).should_receive('listdir').with_args( + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' + ).and_return(['file.txt']) + flexmock(module.shutil).should_receive('rmtree') + flexmock(module).should_receive('unmount_snapshot').with_args( + 'umount', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', + ).never() + flexmock(module).should_receive('unmount_snapshot').with_args( + 'umount', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).once() flexmock(module).should_receive('get_snapshots').and_return( ( @@ -900,24 +1099,27 @@ def test_remove_data_source_dumps_with_successful_mount_point_removal_skips_unmo flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots' + '/run/borgmatic/lvm_snapshots/b33f' ).and_return(True) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots/mnt/lvolume1' + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1' ).and_return(True).and_return(False) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/lvm_snapshots/mnt/lvolume2' + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2' ).and_return(True).and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).never() flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).once() flexmock(module).should_receive('get_snapshots').and_return( ( @@ -957,16 +1159,19 @@ def test_remove_data_source_dumps_bails_for_missing_umount_command(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).and_raise(FileNotFoundError) flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).never() flexmock(module).should_receive('get_snapshots').never() flexmock(module).should_receive('remove_snapshot').never() @@ -979,7 +1184,7 @@ def test_remove_data_source_dumps_bails_for_missing_umount_command(): ) -def test_remove_data_source_dumps_bails_for_umount_command_error(): +def test_remove_data_source_dumps_swallows_umount_command_error(): config = {'lvm': {}} flexmock(module).should_receive('get_logical_volumes').and_return( ( @@ -1000,19 +1205,28 @@ def test_remove_data_source_dumps_bails_for_umount_command_error(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).and_raise(module.subprocess.CalledProcessError(1, 'wtf')) flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', - ).never() - flexmock(module).should_receive('get_snapshots').never() - flexmock(module).should_receive('remove_snapshot').never() + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', + ).once() + flexmock(module).should_receive('get_snapshots').and_return( + ( + module.Snapshot('lvolume1_borgmatic-1234', '/dev/lvolume1'), + module.Snapshot('lvolume2_borgmatic-1234', '/dev/lvolume2'), + ), + ) + flexmock(module).should_receive('remove_snapshot').with_args('lvremove', '/dev/lvolume1').once() + flexmock(module).should_receive('remove_snapshot').with_args('lvremove', '/dev/lvolume2').once() module.remove_data_source_dumps( hook_config=config['lvm'], @@ -1043,16 +1257,19 @@ def test_remove_data_source_dumps_bails_for_missing_lvs_command(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).once() flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).once() flexmock(module).should_receive('get_snapshots').and_raise(FileNotFoundError) flexmock(module).should_receive('remove_snapshot').never() @@ -1086,16 +1303,19 @@ def test_remove_data_source_dumps_bails_for_lvs_command_error(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume1', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume1', ).once() flexmock(module).should_receive('unmount_snapshot').with_args( 'umount', - '/run/borgmatic/lvm_snapshots/mnt/lvolume2', + '/run/borgmatic/lvm_snapshots/b33f/mnt/lvolume2', ).once() flexmock(module).should_receive('get_snapshots').and_raise( module.subprocess.CalledProcessError(1, 'wtf') @@ -1133,6 +1353,7 @@ def test_remove_data_source_with_dry_run_skips_snapshot_unmount_and_delete(): ).and_return('/run/borgmatic') flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree').never() flexmock(module).should_receive('unmount_snapshot').never() flexmock(module).should_receive('get_snapshots').and_return( diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py index e21979d6..871c1969 100644 --- a/tests/unit/hooks/data_source/test_mariadb.py +++ b/tests/unit/hooks/data_source/test_mariadb.py @@ -6,31 +6,221 @@ from flexmock import flexmock from borgmatic.hooks.data_source import mariadb as module -def test_database_names_to_dump_passes_through_name(): - extra_environment = flexmock() +def test_parse_extra_options_passes_through_empty_options(): + assert module.parse_extra_options('') == ((), None) - names = module.database_names_to_dump({'name': 'foo'}, extra_environment, dry_run=False) + +def test_parse_extra_options_with_defaults_extra_file_removes_and_and_parses_out_filename(): + assert module.parse_extra_options('--defaults-extra-file=extra.cnf --skip-ssl') == ( + ('--skip-ssl',), + 'extra.cnf', + ) + + +def test_parse_extra_options_without_defaults_extra_file_passes_through_options(): + assert module.parse_extra_options('--skip-ssl --and=stuff') == ( + ('--skip-ssl', '--and=stuff'), + None, + ) + + +def test_make_defaults_file_pipe_without_username_or_password_bails(): + flexmock(module.os).should_receive('pipe').never() + + assert module.make_defaults_file_options(username=None, password=None) == () + + +def test_make_defaults_file_option_with_username_and_password_writes_them_to_file_descriptor(): + read_descriptor = 99 + write_descriptor = flexmock() + + flexmock(module.os).should_receive('pipe').and_return(read_descriptor, write_descriptor) + flexmock(module.os).should_receive('write').with_args( + write_descriptor, b'[client]\nuser=root\npassword="trustsome1"' + ).once() + flexmock(module.os).should_receive('close') + flexmock(module.os).should_receive('set_inheritable') + + assert module.make_defaults_file_options(username='root', password='trustsome1') == ( + '--defaults-extra-file=/dev/fd/99', + ) + + +def test_make_defaults_file_escapes_password_containing_backslash(): + read_descriptor = 99 + write_descriptor = flexmock() + + flexmock(module.os).should_receive('pipe').and_return(read_descriptor, write_descriptor) + flexmock(module.os).should_receive('write').with_args( + write_descriptor, b'[client]\nuser=root\n' + br'password="trust\\nsome1"' + ).once() + flexmock(module.os).should_receive('close') + flexmock(module.os).should_receive('set_inheritable') + + assert module.make_defaults_file_options(username='root', password=r'trust\nsome1') == ( + '--defaults-extra-file=/dev/fd/99', + ) + + +def test_make_defaults_file_pipe_with_only_username_writes_it_to_file_descriptor(): + read_descriptor = 99 + write_descriptor = flexmock() + + flexmock(module.os).should_receive('pipe').and_return(read_descriptor, write_descriptor) + flexmock(module.os).should_receive('write').with_args( + write_descriptor, b'[client]\nuser=root' + ).once() + flexmock(module.os).should_receive('close') + flexmock(module.os).should_receive('set_inheritable') + + assert module.make_defaults_file_options(username='root', password=None) == ( + '--defaults-extra-file=/dev/fd/99', + ) + + +def test_make_defaults_file_pipe_with_only_password_writes_it_to_file_descriptor(): + read_descriptor = 99 + write_descriptor = flexmock() + + flexmock(module.os).should_receive('pipe').and_return(read_descriptor, write_descriptor) + flexmock(module.os).should_receive('write').with_args( + write_descriptor, b'[client]\npassword="trustsome1"' + ).once() + flexmock(module.os).should_receive('close') + flexmock(module.os).should_receive('set_inheritable') + + assert module.make_defaults_file_options(username=None, password='trustsome1') == ( + '--defaults-extra-file=/dev/fd/99', + ) + + +def test_make_defaults_file_option_with_defaults_extra_filename_includes_it_in_file_descriptor(): + read_descriptor = 99 + write_descriptor = flexmock() + + flexmock(module.os).should_receive('pipe').and_return(read_descriptor, write_descriptor) + flexmock(module.os).should_receive('write').with_args( + write_descriptor, b'!include extra.cnf\n[client]\nuser=root\npassword="trustsome1"' + ).once() + flexmock(module.os).should_receive('close') + flexmock(module.os).should_receive('set_inheritable') + + assert module.make_defaults_file_options( + username='root', password='trustsome1', defaults_extra_filename='extra.cnf' + ) == ('--defaults-extra-file=/dev/fd/99',) + + +def test_make_defaults_file_option_with_only_defaults_extra_filename_uses_it_instead_of_file_descriptor(): + flexmock(module.os).should_receive('pipe').never() + + assert module.make_defaults_file_options( + username=None, password=None, defaults_extra_filename='extra.cnf' + ) == ('--defaults-extra-file=extra.cnf',) + + +def test_database_names_to_dump_passes_through_name(): + environment = flexmock() + + names = module.database_names_to_dump( + {'name': 'foo'}, {}, 'root', 'trustsome1', environment, dry_run=False + ) assert names == ('foo',) def test_database_names_to_dump_bails_for_dry_run(): - extra_environment = flexmock() + environment = flexmock() flexmock(module).should_receive('execute_command_and_capture_output').never() - names = module.database_names_to_dump({'name': 'all'}, extra_environment, dry_run=True) + names = module.database_names_to_dump( + {'name': 'all'}, {}, 'root', 'trustsome1', environment, dry_run=True + ) assert names == () def test_database_names_to_dump_queries_mariadb_for_database_names(): - extra_environment = flexmock() + environment = flexmock() + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('mariadb', '--skip-column-names', '--batch', '--execute', 'show schemas'), - extra_environment=extra_environment, + ( + 'mariadb', + '--defaults-extra-file=/dev/fd/99', + '--skip-column-names', + '--batch', + '--execute', + 'show schemas', + ), + environment=environment, ).and_return('foo\nbar\nmysql\n').once() - names = module.database_names_to_dump({'name': 'all'}, extra_environment, dry_run=False) + names = module.database_names_to_dump( + {'name': 'all'}, {}, 'root', 'trustsome1', environment, dry_run=False + ) + + assert names == ('foo', 'bar') + + +def test_database_names_to_dump_runs_mariadb_with_tls(): + environment = flexmock() + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + ( + 'mariadb', + '--defaults-extra-file=/dev/fd/99', + '--ssl', + '--skip-column-names', + '--batch', + '--execute', + 'show schemas', + ), + environment=environment, + ).and_return('foo\nbar\nmysql\n').once() + + names = module.database_names_to_dump( + {'name': 'all', 'tls': True}, {}, 'root', 'trustsome1', environment, dry_run=False + ) + + assert names == ('foo', 'bar') + + +def test_database_names_to_dump_runs_mariadb_without_tls(): + environment = flexmock() + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + ( + 'mariadb', + '--defaults-extra-file=/dev/fd/99', + '--skip-ssl', + '--skip-column-names', + '--batch', + '--execute', + 'show schemas', + ), + environment=environment, + ).and_return('foo\nbar\nmysql\n').once() + + names = module.database_names_to_dump( + {'name': 'all', 'tls': False}, {}, 'root', 'trustsome1', environment, dry_run=False + ) assert names == ('foo', 'bar') @@ -50,6 +240,13 @@ def test_dump_data_sources_dumps_each_database(): databases = [{'name': 'foo'}, {'name': 'bar'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).and_return(None) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( ('bar',) ) @@ -57,9 +254,12 @@ def test_dump_data_sources_dumps_each_database(): for name, process in zip(('foo', 'bar'), processes): flexmock(module).should_receive('execute_dump_command').with_args( database={'name': name}, + config={}, + username=None, + password=None, dump_path=object, database_names=(name,), - extra_environment=object, + environment={'USER': 'root'}, dry_run=object, dry_run_label=object, ).and_return(process).once() @@ -81,15 +281,22 @@ def test_dump_data_sources_dumps_with_password(): database = {'name': 'foo', 'username': 'root', 'password': 'trustsome1'} process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( ('bar',) ) flexmock(module).should_receive('execute_dump_command').with_args( database=database, + config={}, + username='root', + password='trustsome1', dump_path=object, database_names=('foo',), - extra_environment={'MYSQL_PWD': 'trustsome1'}, + environment={'USER': 'root'}, dry_run=object, dry_run_label=object, ).and_return(process).once() @@ -108,12 +315,19 @@ def test_dump_data_sources_dumps_all_databases_at_once(): databases = [{'name': 'all'}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) flexmock(module).should_receive('execute_dump_command').with_args( database={'name': 'all'}, + config={}, + username=None, + password=None, dump_path=object, database_names=('foo', 'bar'), - extra_environment=object, + environment={'USER': 'root'}, dry_run=object, dry_run_label=object, ).and_return(process).once() @@ -132,14 +346,21 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured databases = [{'name': 'all', 'format': 'sql'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).and_return(None) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) for name, process in zip(('foo', 'bar'), processes): flexmock(module).should_receive('execute_dump_command').with_args( database={'name': name, 'format': 'sql'}, + config={}, + username=None, + password=None, dump_path=object, database_names=(name,), - extra_environment=object, + environment={'USER': 'root'}, dry_run=object, dry_run_label=object, ).and_return(process).once() @@ -158,33 +379,50 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured def test_database_names_to_dump_runs_mariadb_with_list_options(): - database = {'name': 'all', 'list_options': '--defaults-extra-file=mariadb.cnf'} + database = {'name': 'all', 'list_options': '--defaults-extra-file=mariadb.cnf --skip-ssl'} + flexmock(module).should_receive('parse_extra_options').and_return( + ('--skip-ssl',), 'mariadb.cnf' + ) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', 'mariadb.cnf' + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ( 'mariadb', - '--defaults-extra-file=mariadb.cnf', + '--defaults-extra-file=/dev/fd/99', + '--skip-ssl', '--skip-column-names', '--batch', '--execute', 'show schemas', ), - extra_environment=None, + environment=None, ).and_return(('foo\nbar')).once() - assert module.database_names_to_dump(database, None, '') == ('foo', 'bar') + assert module.database_names_to_dump(database, {}, 'root', 'trustsome1', None, '') == ( + 'foo', + 'bar', + ) def test_database_names_to_dump_runs_non_default_mariadb_with_list_options(): database = { 'name': 'all', - 'list_options': '--defaults-extra-file=mariadb.cnf', + 'list_options': '--defaults-extra-file=mariadb.cnf --skip-ssl', 'mariadb_command': 'custom_mariadb', } + flexmock(module).should_receive('parse_extra_options').and_return( + ('--skip-ssl',), 'mariadb.cnf' + ) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', 'mariadb.cnf' + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module).should_receive('execute_command_and_capture_output').with_args( - extra_environment=None, + environment=None, full_command=( 'custom_mariadb', # Custom MariaDB command - '--defaults-extra-file=mariadb.cnf', + '--defaults-extra-file=/dev/fd/99', + '--skip-ssl', '--skip-column-names', '--batch', '--execute', @@ -192,34 +430,48 @@ def test_database_names_to_dump_runs_non_default_mariadb_with_list_options(): ), ).and_return(('foo\nbar')).once() - assert module.database_names_to_dump(database, None, '') == ('foo', 'bar') + assert module.database_names_to_dump(database, {}, 'root', 'trustsome1', None, '') == ( + 'foo', + 'bar', + ) def test_execute_dump_command_runs_mariadb_dump(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mariadb-dump', + '--defaults-extra-file=/dev/fd/99', '--add-drop-database', '--databases', 'foo', '--result-file', 'dump', ), - extra_environment=None, + environment=None, run_to_completion=False, ).and_return(process).once() assert ( module.execute_dump_command( database={'name': 'foo'}, + config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=False, dry_run_label='', ) @@ -231,26 +483,37 @@ def test_execute_dump_command_runs_mariadb_dump_without_add_drop_database(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mariadb-dump', + '--defaults-extra-file=/dev/fd/99', '--databases', 'foo', '--result-file', 'dump', ), - extra_environment=None, + environment=None, run_to_completion=False, ).and_return(process).once() assert ( module.execute_dump_command( database={'name': 'foo', 'add_drop_database': False}, + config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=False, dry_run_label='', ) @@ -262,11 +525,19 @@ def test_execute_dump_command_runs_mariadb_dump_with_hostname_and_port(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mariadb-dump', + '--defaults-extra-file=/dev/fd/99', '--add-drop-database', '--host', 'database.example.org', @@ -279,16 +550,107 @@ def test_execute_dump_command_runs_mariadb_dump_with_hostname_and_port(): '--result-file', 'dump', ), - extra_environment=None, + environment=None, run_to_completion=False, ).and_return(process).once() assert ( module.execute_dump_command( database={'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}, + config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, + dry_run=False, + dry_run_label='', + ) + == process + ) + + +def test_execute_dump_command_runs_mariadb_dump_with_tls(): + process = flexmock() + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'mariadb-dump', + '--defaults-extra-file=/dev/fd/99', + '--add-drop-database', + '--ssl', + '--databases', + 'foo', + '--result-file', + 'dump', + ), + environment=None, + run_to_completion=False, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={'name': 'foo', 'tls': True}, + config={}, + username='root', + password='trustsome1', + dump_path=flexmock(), + database_names=('foo',), + environment=None, + dry_run=False, + dry_run_label='', + ) + == process + ) + + +def test_execute_dump_command_runs_mariadb_dump_without_tls(): + process = flexmock() + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'mariadb-dump', + '--defaults-extra-file=/dev/fd/99', + '--add-drop-database', + '--skip-ssl', + '--databases', + 'foo', + '--result-file', + 'dump', + ), + environment=None, + run_to_completion=False, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={'name': 'foo', 'tls': False}, + config={}, + username='root', + password='trustsome1', + dump_path=flexmock(), + database_names=('foo',), + environment=None, dry_run=False, dry_run_label='', ) @@ -300,29 +662,38 @@ def test_execute_dump_command_runs_mariadb_dump_with_username_and_password(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mariadb-dump', + '--defaults-extra-file=/dev/fd/99', '--add-drop-database', - '--user', - 'root', '--databases', 'foo', '--result-file', 'dump', ), - extra_environment={'MYSQL_PWD': 'trustsome1'}, + environment={}, run_to_completion=False, ).and_return(process).once() assert ( module.execute_dump_command( database={'name': 'foo', 'username': 'root', 'password': 'trustsome1'}, + config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), - extra_environment={'MYSQL_PWD': 'trustsome1'}, + environment={}, dry_run=False, dry_run_label='', ) @@ -334,11 +705,19 @@ def test_execute_dump_command_runs_mariadb_dump_with_options(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return(('--stuff=such',), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mariadb-dump', + '--defaults-extra-file=/dev/fd/99', '--stuff=such', '--add-drop-database', '--databases', @@ -346,16 +725,19 @@ def test_execute_dump_command_runs_mariadb_dump_with_options(): '--result-file', 'dump', ), - extra_environment=None, + environment=None, run_to_completion=False, ).and_return(process).once() assert ( module.execute_dump_command( database={'name': 'foo', 'options': '--stuff=such'}, + config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=False, dry_run_label='', ) @@ -367,11 +749,19 @@ def test_execute_dump_command_runs_non_default_mariadb_dump_with_options(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return(('--stuff=such',), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'custom_mariadb_dump', # Custom MariaDB dump command + '--defaults-extra-file=/dev/fd/99', '--stuff=such', '--add-drop-database', '--databases', @@ -379,7 +769,7 @@ def test_execute_dump_command_runs_non_default_mariadb_dump_with_options(): '--result-file', 'dump', ), - extra_environment=None, + environment=None, run_to_completion=False, ).and_return(process).once() @@ -390,9 +780,12 @@ def test_execute_dump_command_runs_non_default_mariadb_dump_with_options(): 'mariadb_dump_command': 'custom_mariadb_dump', 'options': '--stuff=such', }, # Custom MariaDB dump command specified + config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=False, dry_run_label='', ) @@ -403,15 +796,22 @@ def test_execute_dump_command_runs_non_default_mariadb_dump_with_options(): def test_execute_dump_command_with_duplicate_dump_skips_mariadb_dump(): flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() assert ( module.execute_dump_command( database={'name': 'foo'}, + config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=True, dry_run_label='SO DRY', ) @@ -422,6 +822,13 @@ def test_execute_dump_command_with_duplicate_dump_skips_mariadb_dump(): def test_execute_dump_command_with_dry_run_skips_mariadb_dump(): flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').never() @@ -429,9 +836,12 @@ def test_execute_dump_command_with_dry_run_skips_mariadb_dump(): assert ( module.execute_dump_command( database={'name': 'foo'}, + config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=True, dry_run_label='SO DRY', ) @@ -442,6 +852,10 @@ def test_execute_dump_command_with_dry_run_skips_mariadb_dump(): def test_dump_data_sources_errors_for_missing_all_databases(): databases = [{'name': 'all'}] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( 'databases/localhost/all' ) @@ -461,6 +875,10 @@ def test_dump_data_sources_errors_for_missing_all_databases(): def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run(): databases = [{'name': 'all'}] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( 'databases/localhost/all' ) @@ -483,12 +901,20 @@ def test_restore_data_source_dump_runs_mariadb_to_restore(): hook_config = [{'name': 'foo'}, {'name': 'bar'}] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + None, None, None + ).and_return(()) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mariadb', '--batch'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment=None, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -511,12 +937,20 @@ def test_restore_data_source_dump_runs_mariadb_with_options(): hook_config = [{'name': 'foo', 'restore_options': '--harder'}] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return(('--harder',), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + None, None, None + ).and_return(()) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( - ('mariadb', '--batch', '--harder'), + ('mariadb', '--harder', '--batch'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment=None, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -541,12 +975,20 @@ def test_restore_data_source_dump_runs_non_default_mariadb_with_options(): ] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return(('--harder',), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + None, None, None + ).and_return(()) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( - ('custom_mariadb', '--batch', '--harder'), + ('custom_mariadb', '--harder', '--batch'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment=None, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -569,6 +1011,14 @@ def test_restore_data_source_dump_runs_mariadb_with_hostname_and_port(): hook_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + None, None, None + ).and_return(()) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mariadb', @@ -583,7 +1033,87 @@ def test_restore_data_source_dump_runs_mariadb_with_hostname_and_port(): processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment=None, + environment={'USER': 'root'}, + ).once() + + module.restore_data_source_dump( + hook_config, + {}, + data_source=hook_config[0], + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + borgmatic_runtime_directory='/run/borgmatic', + ) + + +def test_restore_data_source_dump_runs_mariadb_with_tls(): + hook_config = [{'name': 'foo', 'tls': True}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + None, None, None + ).and_return(()) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'mariadb', + '--batch', + '--ssl', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + environment={'USER': 'root'}, + ).once() + + module.restore_data_source_dump( + hook_config, + {}, + data_source=hook_config[0], + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + borgmatic_runtime_directory='/run/borgmatic', + ) + + +def test_restore_data_source_dump_runs_mariadb_without_tls(): + hook_config = [{'name': 'foo', 'tls': False}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + None, None, None + ).and_return(()) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'mariadb', + '--batch', + '--skip-ssl', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -606,12 +1136,20 @@ def test_restore_data_source_dump_runs_mariadb_with_username_and_password(): hook_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'root', 'trustsome1', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( - ('mariadb', '--batch', '--user', 'root'), + ('mariadb', '--defaults-extra-file=/dev/fd/99', '--batch'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'MYSQL_PWD': 'trustsome1'}, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -644,9 +1182,18 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ ] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'cliusername', 'clipassword', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mariadb', + '--defaults-extra-file=/dev/fd/99', '--batch', '--host', 'clihost', @@ -654,13 +1201,11 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ 'cliport', '--protocol', 'tcp', - '--user', - 'cliusername', ), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'MYSQL_PWD': 'clipassword'}, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -687,17 +1232,28 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ 'password': 'trustsome1', 'hostname': 'dbhost', 'port': 'dbport', + 'tls': True, 'restore_username': 'restoreuser', 'restore_password': 'restorepass', 'restore_hostname': 'restorehost', 'restore_port': 'restoreport', + 'restore_tls': False, } ] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + 'restoreuser', 'restorepass', None + ).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mariadb', + '--defaults-extra-file=/dev/fd/99', '--batch', '--host', 'restorehost', @@ -705,13 +1261,12 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ 'restoreport', '--protocol', 'tcp', - '--user', - 'restoreuser', + '--skip-ssl', ), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'MYSQL_PWD': 'restorepass'}, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -733,6 +1288,14 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ def test_restore_data_source_dump_with_dry_run_skips_restore(): hook_config = [{'name': 'foo'}] + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('parse_extra_options').and_return((), None) + flexmock(module).should_receive('make_defaults_file_options').with_args( + None, None, None + ).and_return(()) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').never() module.restore_data_source_dump( diff --git a/tests/unit/hooks/data_source/test_mongodb.py b/tests/unit/hooks/data_source/test_mongodb.py index 1d2c0c7b..3cd0aff3 100644 --- a/tests/unit/hooks/data_source/test_mongodb.py +++ b/tests/unit/hooks/data_source/test_mongodb.py @@ -124,6 +124,12 @@ def test_dump_data_sources_runs_mongodump_with_username_and_password(): flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( 'databases/localhost/foo' ) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_password_config_file').with_args('trustsome1').and_return( + '/dev/fd/99' + ) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -131,8 +137,8 @@ def test_dump_data_sources_runs_mongodump_with_username_and_password(): 'mongodump', '--username', 'mongo', - '--password', - 'trustsome1', + '--config', + '/dev/fd/99', '--authenticationDatabase', 'admin', '--db', @@ -240,10 +246,29 @@ def test_dump_data_sources_runs_mongodumpall_for_all_databases(): ) == [process] +def test_make_password_config_file_writes_password_to_pipe(): + read_file_descriptor = 99 + write_file_descriptor = flexmock() + + flexmock(module.os).should_receive('pipe').and_return( + (read_file_descriptor, write_file_descriptor) + ) + flexmock(module.os).should_receive('write').with_args( + write_file_descriptor, b'password: trustsome1' + ).once() + flexmock(module.os).should_receive('close') + flexmock(module.os).should_receive('set_inheritable') + + assert module.make_password_config_file('trustsome1') == '/dev/fd/99' + + def test_build_dump_command_with_username_injection_attack_gets_escaped(): database = {'name': 'test', 'username': 'bob; naughty-command'} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) - command = module.build_dump_command(database, dump_filename='test', dump_format='archive') + command = module.build_dump_command(database, {}, dump_filename='test', dump_format='archive') assert "'bob; naughty-command'" in command @@ -254,6 +279,9 @@ def test_restore_data_source_dump_runs_mongorestore(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ['mongorestore', '--archive', '--drop'], processes=[extract_process], @@ -285,6 +313,9 @@ def test_restore_data_source_dump_runs_mongorestore_with_hostname_and_port(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( [ 'mongorestore', @@ -330,6 +361,12 @@ def test_restore_data_source_dump_runs_mongorestore_with_username_and_password() flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_password_config_file').with_args('trustsome1').and_return( + '/dev/fd/99' + ) flexmock(module).should_receive('execute_command_with_processes').with_args( [ 'mongorestore', @@ -337,8 +374,8 @@ def test_restore_data_source_dump_runs_mongorestore_with_username_and_password() '--drop', '--username', 'mongo', - '--password', - 'trustsome1', + '--config', + '/dev/fd/99', '--authenticationDatabase', 'admin', ], @@ -381,6 +418,12 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_password_config_file').with_args( + 'clipassword' + ).and_return('/dev/fd/99') flexmock(module).should_receive('execute_command_with_processes').with_args( [ 'mongorestore', @@ -392,8 +435,8 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ 'cliport', '--username', 'cliusername', - '--password', - 'clipassword', + '--config', + '/dev/fd/99', '--authenticationDatabase', 'admin', ], @@ -436,6 +479,12 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_password_config_file').with_args( + 'restorepass' + ).and_return('/dev/fd/99') flexmock(module).should_receive('execute_command_with_processes').with_args( [ 'mongorestore', @@ -447,8 +496,8 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ 'restoreport', '--username', 'restoreuser', - '--password', - 'restorepass', + '--config', + '/dev/fd/99', '--authenticationDatabase', 'admin', ], @@ -479,6 +528,9 @@ def test_restore_data_source_dump_runs_mongorestore_with_options(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ['mongorestore', '--archive', '--drop', '--harder'], processes=[extract_process], @@ -508,6 +560,9 @@ def test_restore_databases_dump_runs_mongorestore_with_schemas(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( [ 'mongorestore', @@ -545,6 +600,9 @@ def test_restore_data_source_dump_runs_psql_for_all_database_dump(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ['mongorestore', '--archive'], processes=[extract_process], @@ -573,6 +631,9 @@ def test_restore_data_source_dump_with_dry_run_skips_restore(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').never() module.restore_data_source_dump( @@ -596,6 +657,9 @@ def test_restore_data_source_dump_without_extract_process_restores_from_disk(): flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('/dump/path') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_with_processes').with_args( ['mongorestore', '--dir', '/dump/path', '--drop'], processes=[], @@ -617,3 +681,135 @@ def test_restore_data_source_dump_without_extract_process_restores_from_disk(): }, borgmatic_runtime_directory='/run/borgmatic', ) + + +def test_dump_data_sources_uses_custom_mongodump_command(): + flexmock(module.borgmatic.hooks.command).should_receive('Before_after_hooks').and_return( + flexmock() + ) + databases = [{'name': 'foo', 'mongodump_command': 'custom_mongodump'}] + process = flexmock() + flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( + 'databases/localhost/foo' + ) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'custom_mongodump', + '--db', + 'foo', + '--archive', + '>', + 'databases/localhost/foo', + ), + shell=True, + run_to_completion=False, + ).and_return(process).once() + + assert module.dump_data_sources( + databases, + {}, + config_paths=('test.yaml',), + borgmatic_runtime_directory='/run/borgmatic', + patterns=[], + dry_run=False, + ) == [process] + + +def test_build_dump_command_prevents_shell_injection(): + database = { + 'name': 'testdb; rm -rf /', # Malicious input + 'hostname': 'localhost', + 'port': 27017, + 'username': 'user', + 'password': 'password', + 'mongodump_command': 'mongodump', + 'options': '--gzip', + } + config = {} + dump_filename = '/path/to/dump' + dump_format = 'archive' + + command = module.build_dump_command(database, config, dump_filename, dump_format) + + # Ensure the malicious input is properly escaped and does not execute + assert 'testdb; rm -rf /' not in command + assert any( + 'testdb' in part for part in command + ) # Check if 'testdb' is in any part of the tuple + + +def test_restore_data_source_dump_uses_custom_mongorestore_command(): + hook_config = [ + { + 'name': 'foo', + 'mongorestore_command': 'custom_mongorestore', + 'schemas': None, + 'restore_options': '--gzip', + } + ] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('make_dump_path') + flexmock(module.dump).should_receive('make_data_source_dump_filename') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('execute_command_with_processes').with_args( + [ + 'custom_mongorestore', # Should use custom command instead of default + '--archive', + '--drop', + '--gzip', # Should include restore options + ], + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + ).once() + + module.restore_data_source_dump( + hook_config, + {}, + data_source=hook_config[0], + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + borgmatic_runtime_directory='/run/borgmatic', + ) + + +def test_build_restore_command_prevents_shell_injection(): + database = { + 'name': 'testdb; rm -rf /', # Malicious input + 'restore_hostname': 'localhost', + 'restore_port': 27017, + 'restore_username': 'user', + 'restore_password': 'password', + 'mongorestore_command': 'mongorestore', + 'restore_options': '--gzip', + } + config = {} + dump_filename = '/path/to/dump' + connection_params = { + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + } + extract_process = None + + command = module.build_restore_command( + extract_process, database, config, dump_filename, connection_params + ) + + # print(command) + # Ensure the malicious input is properly escaped and does not execute + assert 'rm -rf /' not in command + assert ';' not in command diff --git a/tests/unit/hooks/data_source/test_mysql.py b/tests/unit/hooks/data_source/test_mysql.py index 5eb46e35..63bb7a72 100644 --- a/tests/unit/hooks/data_source/test_mysql.py +++ b/tests/unit/hooks/data_source/test_mysql.py @@ -7,30 +7,117 @@ from borgmatic.hooks.data_source import mysql as module def test_database_names_to_dump_passes_through_name(): - extra_environment = flexmock() + environment = flexmock() - names = module.database_names_to_dump({'name': 'foo'}, extra_environment, dry_run=False) + names = module.database_names_to_dump( + {'name': 'foo'}, {}, 'root', 'trustsome1', environment, dry_run=False + ) assert names == ('foo',) def test_database_names_to_dump_bails_for_dry_run(): - extra_environment = flexmock() + environment = flexmock() + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_and_capture_output').never() - names = module.database_names_to_dump({'name': 'all'}, extra_environment, dry_run=True) + names = module.database_names_to_dump( + {'name': 'all'}, {}, 'root', 'trustsome1', environment, dry_run=True + ) assert names == () def test_database_names_to_dump_queries_mysql_for_database_names(): - extra_environment = flexmock() + environment = flexmock() + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module).should_receive('execute_command_and_capture_output').with_args( - ('mysql', '--skip-column-names', '--batch', '--execute', 'show schemas'), - extra_environment=extra_environment, + ( + 'mysql', + '--defaults-extra-file=/dev/fd/99', + '--skip-column-names', + '--batch', + '--execute', + 'show schemas', + ), + environment=environment, ).and_return('foo\nbar\nmysql\n').once() - names = module.database_names_to_dump({'name': 'all'}, extra_environment, dry_run=False) + names = module.database_names_to_dump( + {'name': 'all'}, {}, 'root', 'trustsome1', environment, dry_run=False + ) + + assert names == ('foo', 'bar') + + +def test_database_names_to_dump_runs_mysql_with_tls(): + environment = flexmock() + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + ( + 'mysql', + '--defaults-extra-file=/dev/fd/99', + '--ssl', + '--skip-column-names', + '--batch', + '--execute', + 'show schemas', + ), + environment=environment, + ).and_return('foo\nbar\nmysql\n').once() + + names = module.database_names_to_dump( + {'name': 'all', 'tls': True}, {}, 'root', 'trustsome1', environment, dry_run=False + ) + + assert names == ('foo', 'bar') + + +def test_database_names_to_dump_runs_mysql_without_tls(): + environment = flexmock() + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module).should_receive('execute_command_and_capture_output').with_args( + ( + 'mysql', + '--defaults-extra-file=/dev/fd/99', + '--skip-ssl', + '--skip-column-names', + '--batch', + '--execute', + 'show schemas', + ), + environment=environment, + ).and_return('foo\nbar\nmysql\n').once() + + names = module.database_names_to_dump( + {'name': 'all', 'tls': False}, {}, 'root', 'trustsome1', environment, dry_run=False + ) assert names == ('foo', 'bar') @@ -50,6 +137,10 @@ def test_dump_data_sources_dumps_each_database(): databases = [{'name': 'foo'}, {'name': 'bar'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).and_return(None) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( ('bar',) ) @@ -57,9 +148,12 @@ def test_dump_data_sources_dumps_each_database(): for name, process in zip(('foo', 'bar'), processes): flexmock(module).should_receive('execute_dump_command').with_args( database={'name': name}, + config={}, + username=None, + password=None, dump_path=object, database_names=(name,), - extra_environment=object, + environment={'USER': 'root'}, dry_run=object, dry_run_label=object, ).and_return(process).once() @@ -81,15 +175,22 @@ def test_dump_data_sources_dumps_with_password(): database = {'name': 'foo', 'username': 'root', 'password': 'trustsome1'} process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( ('bar',) ) flexmock(module).should_receive('execute_dump_command').with_args( database=database, + config={}, + username='root', + password='trustsome1', dump_path=object, database_names=('foo',), - extra_environment={'MYSQL_PWD': 'trustsome1'}, + environment={'USER': 'root'}, dry_run=object, dry_run_label=object, ).and_return(process).once() @@ -108,12 +209,19 @@ def test_dump_data_sources_dumps_all_databases_at_once(): databases = [{'name': 'all'}] process = flexmock() flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).and_return(None) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) flexmock(module).should_receive('execute_dump_command').with_args( database={'name': 'all'}, + config={}, + username=None, + password=None, dump_path=object, database_names=('foo', 'bar'), - extra_environment=object, + environment={'USER': 'root'}, dry_run=object, dry_run_label=object, ).and_return(process).once() @@ -132,14 +240,21 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured databases = [{'name': 'all', 'format': 'sql'}] processes = [flexmock(), flexmock()] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).and_return(None) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) for name, process in zip(('foo', 'bar'), processes): flexmock(module).should_receive('execute_dump_command').with_args( database={'name': name, 'format': 'sql'}, + config={}, + username=None, + password=None, dump_path=object, database_names=(name,), - extra_environment=object, + environment={'USER': 'root'}, dry_run=object, dry_run_label=object, ).and_return(process).once() @@ -158,33 +273,50 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured def test_database_names_to_dump_runs_mysql_with_list_options(): - database = {'name': 'all', 'list_options': '--defaults-extra-file=my.cnf'} + database = {'name': 'all', 'list_options': '--defaults-extra-file=my.cnf --skip-ssl'} + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return(('--skip-ssl',), 'my.cnf') + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', 'my.cnf').and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ( 'mysql', - '--defaults-extra-file=my.cnf', + '--defaults-extra-file=/dev/fd/99', + '--skip-ssl', '--skip-column-names', '--batch', '--execute', 'show schemas', ), - extra_environment=None, + environment=None, ).and_return(('foo\nbar')).once() - assert module.database_names_to_dump(database, None, '') == ('foo', 'bar') + assert module.database_names_to_dump(database, {}, 'root', 'trustsome1', None, '') == ( + 'foo', + 'bar', + ) def test_database_names_to_dump_runs_non_default_mysql_with_list_options(): database = { 'name': 'all', - 'list_options': '--defaults-extra-file=my.cnf', + 'list_options': '--defaults-extra-file=my.cnf --skip-ssl', 'mysql_command': 'custom_mysql', } + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return(('--skip-ssl',), 'my.cnf') + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', 'my.cnf').and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module).should_receive('execute_command_and_capture_output').with_args( - extra_environment=None, + environment=None, full_command=( 'custom_mysql', # Custom MySQL command - '--defaults-extra-file=my.cnf', + '--defaults-extra-file=/dev/fd/99', + '--skip-ssl', '--skip-column-names', '--batch', '--execute', @@ -192,34 +324,50 @@ def test_database_names_to_dump_runs_non_default_mysql_with_list_options(): ), ).and_return(('foo\nbar')).once() - assert module.database_names_to_dump(database, None, '') == ('foo', 'bar') + assert module.database_names_to_dump(database, {}, 'root', 'trustsome1', None, '') == ( + 'foo', + 'bar', + ) def test_execute_dump_command_runs_mysqldump(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mysqldump', + '--defaults-extra-file=/dev/fd/99', '--add-drop-database', '--databases', 'foo', '--result-file', 'dump', ), - extra_environment=None, + environment=None, run_to_completion=False, ).and_return(process).once() assert ( module.execute_dump_command( database={'name': 'foo'}, + config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=False, dry_run_label='', ) @@ -231,26 +379,39 @@ def test_execute_dump_command_runs_mysqldump_without_add_drop_database(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mysqldump', + '--defaults-extra-file=/dev/fd/99', '--databases', 'foo', '--result-file', 'dump', ), - extra_environment=None, + environment=None, run_to_completion=False, ).and_return(process).once() assert ( module.execute_dump_command( database={'name': 'foo', 'add_drop_database': False}, + config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=False, dry_run_label='', ) @@ -262,11 +423,21 @@ def test_execute_dump_command_runs_mysqldump_with_hostname_and_port(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mysqldump', + '--defaults-extra-file=/dev/fd/99', '--add-drop-database', '--host', 'database.example.org', @@ -279,16 +450,111 @@ def test_execute_dump_command_runs_mysqldump_with_hostname_and_port(): '--result-file', 'dump', ), - extra_environment=None, + environment=None, run_to_completion=False, ).and_return(process).once() assert ( module.execute_dump_command( database={'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}, + config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, + dry_run=False, + dry_run_label='', + ) + == process + ) + + +def test_execute_dump_command_runs_mysqldump_with_tls(): + process = flexmock() + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'mysqldump', + '--defaults-extra-file=/dev/fd/99', + '--add-drop-database', + '--ssl', + '--databases', + 'foo', + '--result-file', + 'dump', + ), + environment=None, + run_to_completion=False, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={'name': 'foo', 'tls': True}, + config={}, + username='root', + password='trustsome1', + dump_path=flexmock(), + database_names=('foo',), + environment=None, + dry_run=False, + dry_run_label='', + ) + == process + ) + + +def test_execute_dump_command_runs_mysqldump_without_tls(): + process = flexmock() + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'mysqldump', + '--defaults-extra-file=/dev/fd/99', + '--add-drop-database', + '--skip-ssl', + '--databases', + 'foo', + '--result-file', + 'dump', + ), + environment=None, + run_to_completion=False, + ).and_return(process).once() + + assert ( + module.execute_dump_command( + database={'name': 'foo', 'tls': False}, + config={}, + username='root', + password='trustsome1', + dump_path=flexmock(), + database_names=('foo',), + environment=None, dry_run=False, dry_run_label='', ) @@ -300,29 +566,40 @@ def test_execute_dump_command_runs_mysqldump_with_username_and_password(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mysqldump', + '--defaults-extra-file=/dev/fd/99', '--add-drop-database', - '--user', - 'root', '--databases', 'foo', '--result-file', 'dump', ), - extra_environment={'MYSQL_PWD': 'trustsome1'}, + environment={}, run_to_completion=False, ).and_return(process).once() assert ( module.execute_dump_command( database={'name': 'foo', 'username': 'root', 'password': 'trustsome1'}, + config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), - extra_environment={'MYSQL_PWD': 'trustsome1'}, + environment={}, dry_run=False, dry_run_label='', ) @@ -334,11 +611,21 @@ def test_execute_dump_command_runs_mysqldump_with_options(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return(('--stuff=such',), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'mysqldump', + '--defaults-extra-file=/dev/fd/99', '--stuff=such', '--add-drop-database', '--databases', @@ -346,16 +633,19 @@ def test_execute_dump_command_runs_mysqldump_with_options(): '--result-file', 'dump', ), - extra_environment=None, + environment=None, run_to_completion=False, ).and_return(process).once() assert ( module.execute_dump_command( database={'name': 'foo', 'options': '--stuff=such'}, + config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=False, dry_run_label='', ) @@ -367,18 +657,28 @@ def test_execute_dump_command_runs_non_default_mysqldump(): process = flexmock() flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ( 'custom_mysqldump', # Custom MySQL dump command + '--defaults-extra-file=/dev/fd/99', '--add-drop-database', '--databases', 'foo', '--result-file', 'dump', ), - extra_environment=None, + environment=None, run_to_completion=False, ).and_return(process).once() @@ -388,9 +688,12 @@ def test_execute_dump_command_runs_non_default_mysqldump(): 'name': 'foo', 'mysql_dump_command': 'custom_mysqldump', }, # Custom MySQL dump command specified + config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=False, dry_run_label='', ) @@ -401,15 +704,24 @@ def test_execute_dump_command_runs_non_default_mysqldump(): def test_execute_dump_command_with_duplicate_dump_skips_mysqldump(): flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() assert ( module.execute_dump_command( database={'name': 'foo'}, + config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=True, dry_run_label='SO DRY', ) @@ -420,6 +732,15 @@ def test_execute_dump_command_with_duplicate_dump_skips_mysqldump(): def test_execute_dump_command_with_dry_run_skips_mysqldump(): flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('dump') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').never() @@ -427,9 +748,12 @@ def test_execute_dump_command_with_dry_run_skips_mysqldump(): assert ( module.execute_dump_command( database={'name': 'foo'}, + config={}, + username='root', + password='trustsome1', dump_path=flexmock(), database_names=('foo',), - extra_environment=None, + environment=None, dry_run=True, dry_run_label='SO DRY', ) @@ -440,6 +764,10 @@ def test_execute_dump_command_with_dry_run_skips_mysqldump(): def test_dump_data_sources_errors_for_missing_all_databases(): databases = [{'name': 'all'}] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( 'databases/localhost/all' ) @@ -459,6 +787,10 @@ def test_dump_data_sources_errors_for_missing_all_databases(): def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run(): databases = [{'name': 'all'}] flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( 'databases/localhost/all' ) @@ -481,12 +813,22 @@ def test_restore_data_source_dump_runs_mysql_to_restore(): hook_config = [{'name': 'foo'}, {'name': 'bar'}] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ('mysql', '--batch'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment=None, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -509,12 +851,22 @@ def test_restore_data_source_dump_runs_mysql_with_options(): hook_config = [{'name': 'foo', 'restore_options': '--harder'}] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return(('--harder',), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( - ('mysql', '--batch', '--harder'), + ('mysql', '--harder', '--batch'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment=None, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -537,12 +889,22 @@ def test_restore_data_source_dump_runs_non_default_mysql_with_options(): hook_config = [{'name': 'foo', 'mysql_command': 'custom_mysql', 'restore_options': '--harder'}] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return(('--harder',), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( - ('custom_mysql', '--batch', '--harder'), + ('custom_mysql', '--harder', '--batch'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment=None, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -565,6 +927,16 @@ def test_restore_data_source_dump_runs_mysql_with_hostname_and_port(): hook_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mysql', @@ -579,7 +951,91 @@ def test_restore_data_source_dump_runs_mysql_with_hostname_and_port(): processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment=None, + environment={'USER': 'root'}, + ).once() + + module.restore_data_source_dump( + hook_config, + {}, + data_source=hook_config[0], + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + borgmatic_runtime_directory='/run/borgmatic', + ) + + +def test_restore_data_source_dump_runs_mysql_with_tls(): + hook_config = [{'name': 'foo', 'tls': True}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'mysql', + '--batch', + '--ssl', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + environment={'USER': 'root'}, + ).once() + + module.restore_data_source_dump( + hook_config, + {}, + data_source=hook_config[0], + dry_run=False, + extract_process=extract_process, + connection_params={ + 'hostname': None, + 'port': None, + 'username': None, + 'password': None, + }, + borgmatic_runtime_directory='/run/borgmatic', + ) + + +def test_restore_data_source_dump_runs_mysql_without_tls(): + hook_config = [{'name': 'foo', 'tls': False}] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'mysql', + '--batch', + '--skip-ssl', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -602,12 +1058,22 @@ def test_restore_data_source_dump_runs_mysql_with_username_and_password(): hook_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('root', 'trustsome1', None).and_return(('--defaults-extra-file=/dev/fd/99',)) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( - ('mysql', '--batch', '--user', 'root'), + ('mysql', '--defaults-extra-file=/dev/fd/99', '--batch'), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'MYSQL_PWD': 'trustsome1'}, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -640,9 +1106,22 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ ] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('cliusername', 'clipassword', None).and_return( + ('--defaults-extra-file=/dev/fd/99',) + ) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mysql', + '--defaults-extra-file=/dev/fd/99', '--batch', '--host', 'clihost', @@ -650,13 +1129,11 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ 'cliport', '--protocol', 'tcp', - '--user', - 'cliusername', ), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'MYSQL_PWD': 'clipassword'}, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -683,17 +1160,32 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ 'password': 'trustsome1', 'hostname': 'dbhost', 'port': 'dbport', + 'tls': True, 'restore_username': 'restoreuser', 'restore_password': 'restorepass', 'restore_hostname': 'restorehost', 'restore_port': 'restoreport', + 'restore_tls': False, } ] extract_process = flexmock(stdout=flexmock()) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args('restoreuser', 'restorepass', None).and_return( + ('--defaults-extra-file=/dev/fd/99',) + ) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').with_args( ( 'mysql', + '--defaults-extra-file=/dev/fd/99', '--batch', '--host', 'restorehost', @@ -701,13 +1193,12 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ 'restoreport', '--protocol', 'tcp', - '--user', - 'restoreuser', + '--skip-ssl', ), processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'MYSQL_PWD': 'restorepass'}, + environment={'USER': 'root'}, ).once() module.restore_data_source_dump( @@ -729,6 +1220,16 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ def test_restore_data_source_dump_with_dry_run_skips_restore(): hook_config = [{'name': 'foo'}] + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'parse_extra_options' + ).and_return((), None) + flexmock(module.borgmatic.hooks.data_source.mariadb).should_receive( + 'make_defaults_file_options' + ).with_args(None, None, None).and_return(()) + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) flexmock(module).should_receive('execute_command_with_processes').never() module.restore_data_source_dump( diff --git a/tests/unit/hooks/data_source/test_postgresql.py b/tests/unit/hooks/data_source/test_postgresql.py index 845859f7..f1589a6e 100644 --- a/tests/unit/hooks/data_source/test_postgresql.py +++ b/tests/unit/hooks/data_source/test_postgresql.py @@ -6,7 +6,7 @@ from flexmock import flexmock from borgmatic.hooks.data_source import postgresql as module -def test_make_extra_environment_maps_options_to_environment(): +def test_make_environment_maps_options_to_environment(): database = { 'name': 'foo', 'password': 'pass', @@ -17,6 +17,7 @@ def test_make_extra_environment_maps_options_to_environment(): 'ssl_crl': 'crl.crl', } expected = { + 'USER': 'root', 'PGPASSWORD': 'pass', 'PGSSLMODE': 'require', 'PGSSLCERT': 'cert.crt', @@ -24,72 +25,84 @@ def test_make_extra_environment_maps_options_to_environment(): 'PGSSLROOTCERT': 'root.crt', 'PGSSLCRL': 'crl.crl', } + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) - extra_env = module.make_extra_environment(database) - - assert extra_env == expected + assert module.make_environment(database, {}) == expected -def test_make_extra_environment_with_cli_password_sets_correct_password(): +def test_make_environment_with_cli_password_sets_correct_password(): database = {'name': 'foo', 'restore_password': 'trustsome1', 'password': 'anotherpassword'} + flexmock(module.os).should_receive('environ').and_return({'USER': 'root'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) - extra = module.make_extra_environment( - database, restore_connection_params={'password': 'clipassword'} + environment = module.make_environment( + database, {}, restore_connection_params={'password': 'clipassword'} ) - assert extra['PGPASSWORD'] == 'clipassword' + assert environment['PGPASSWORD'] == 'clipassword' -def test_make_extra_environment_without_cli_password_or_configured_password_does_not_set_password(): +def test_make_environment_without_cli_password_or_configured_password_does_not_set_password(): database = {'name': 'foo'} - extra = module.make_extra_environment( - database, restore_connection_params={'username': 'someone'} + environment = module.make_environment( + database, {}, restore_connection_params={'username': 'someone'} ) - assert 'PGPASSWORD' not in extra + assert 'PGPASSWORD' not in environment -def test_make_extra_environment_without_ssl_mode_does_not_set_ssl_mode(): +def test_make_environment_without_ssl_mode_does_not_set_ssl_mode(): database = {'name': 'foo'} - extra = module.make_extra_environment(database) + environment = module.make_environment(database, {}) - assert 'PGSSLMODE' not in extra + assert 'PGSSLMODE' not in environment def test_database_names_to_dump_passes_through_individual_database_name(): database = {'name': 'foo'} - assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ('foo',) + assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('foo',) def test_database_names_to_dump_passes_through_individual_database_name_with_format(): database = {'name': 'foo', 'format': 'custom'} - assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ('foo',) + assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('foo',) def test_database_names_to_dump_passes_through_all_without_format(): database = {'name': 'all'} - assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ('all',) + assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('all',) def test_database_names_to_dump_with_all_and_format_and_dry_run_bails(): database = {'name': 'all', 'format': 'custom'} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_and_capture_output').never() - assert module.database_names_to_dump(database, flexmock(), dry_run=True) == () + assert module.database_names_to_dump(database, {}, flexmock(), dry_run=True) == () def test_database_names_to_dump_with_all_and_format_lists_databases(): database = {'name': 'all', 'format': 'custom'} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_and_capture_output').and_return( 'foo,test,\nbar,test,"stuff and such"' ) - assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ( + assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ( 'foo', 'bar', ) @@ -97,6 +110,9 @@ def test_database_names_to_dump_with_all_and_format_lists_databases(): def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostname_and_port(): database = {'name': 'all', 'format': 'custom', 'hostname': 'localhost', 'port': 1234} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ( 'psql', @@ -110,10 +126,10 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostnam '--port', '1234', ), - extra_environment=object, + environment=object, ).and_return('foo,test,\nbar,test,"stuff and such"') - assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ( + assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ( 'foo', 'bar', ) @@ -121,6 +137,9 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostnam def test_database_names_to_dump_with_all_and_format_lists_databases_with_username(): database = {'name': 'all', 'format': 'custom', 'username': 'postgres'} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ( 'psql', @@ -132,10 +151,10 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_usernam '--username', 'postgres', ), - extra_environment=object, + environment=object, ).and_return('foo,test,\nbar,test,"stuff and such"') - assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ( + assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ( 'foo', 'bar', ) @@ -143,12 +162,15 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_usernam def test_database_names_to_dump_with_all_and_format_lists_databases_with_options(): database = {'name': 'all', 'format': 'custom', 'list_options': '--harder'} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('psql', '--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only', '--harder'), - extra_environment=object, + environment=object, ).and_return('foo,test,\nbar,test,"stuff and such"') - assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ( + assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ( 'foo', 'bar', ) @@ -156,11 +178,14 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_options def test_database_names_to_dump_with_all_and_format_excludes_particular_databases(): database = {'name': 'all', 'format': 'custom'} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_and_capture_output').and_return( 'foo,test,\ntemplate0,test,blah' ) - assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ('foo',) + assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('foo',) def test_database_names_to_dump_with_all_and_psql_command_uses_custom_command(): @@ -169,6 +194,9 @@ def test_database_names_to_dump_with_all_and_psql_command_uses_custom_command(): 'format': 'custom', 'psql_command': 'docker exec --workdir * mycontainer psql', } + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module).should_receive('execute_command_and_capture_output').with_args( ( 'docker', @@ -183,10 +211,10 @@ def test_database_names_to_dump_with_all_and_psql_command_uses_custom_command(): '--csv', '--tuples-only', ), - extra_environment=object, + environment=object, ).and_return('foo,text').once() - assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ('foo',) + assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('foo',) def test_use_streaming_true_for_any_non_directory_format_databases(): @@ -210,7 +238,7 @@ def test_use_streaming_false_for_no_databases(): def test_dump_data_sources_runs_pg_dump_for_each_database(): databases = [{'name': 'foo'}, {'name': 'bar'}] processes = [flexmock(), flexmock()] - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( ('bar',) @@ -219,6 +247,9 @@ def test_dump_data_sources_runs_pg_dump_for_each_database(): 'databases/localhost/foo' ).and_return('databases/localhost/bar') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') for name, process in zip(('foo', 'bar'), processes): @@ -235,7 +266,7 @@ def test_dump_data_sources_runs_pg_dump_for_each_database(): f'databases/localhost/{name}', ), shell=True, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() @@ -254,7 +285,7 @@ def test_dump_data_sources_runs_pg_dump_for_each_database(): def test_dump_data_sources_raises_when_no_database_names_to_dump(): databases = [{'name': 'foo'}, {'name': 'bar'}] - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module).should_receive('database_names_to_dump').and_return(()) @@ -271,7 +302,7 @@ def test_dump_data_sources_raises_when_no_database_names_to_dump(): def test_dump_data_sources_does_not_raise_when_no_database_names_to_dump(): databases = [{'name': 'foo'}, {'name': 'bar'}] - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module).should_receive('database_names_to_dump').and_return(()) @@ -287,7 +318,7 @@ def test_dump_data_sources_does_not_raise_when_no_database_names_to_dump(): def test_dump_data_sources_with_duplicate_dump_skips_pg_dump(): databases = [{'name': 'foo'}, {'name': 'bar'}] - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( ('bar',) @@ -314,7 +345,7 @@ def test_dump_data_sources_with_duplicate_dump_skips_pg_dump(): def test_dump_data_sources_with_dry_run_skips_pg_dump(): databases = [{'name': 'foo'}, {'name': 'bar'}] - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return( ('bar',) @@ -323,6 +354,9 @@ def test_dump_data_sources_with_dry_run_skips_pg_dump(): 'databases/localhost/foo' ).and_return('databases/localhost/bar') flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() flexmock(module).should_receive('execute_command').never() @@ -342,13 +376,16 @@ def test_dump_data_sources_with_dry_run_skips_pg_dump(): def test_dump_data_sources_runs_pg_dump_with_hostname_and_port(): databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}] process = flexmock() - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( 'databases/database.example.org/foo' ) flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -368,7 +405,7 @@ def test_dump_data_sources_runs_pg_dump_with_hostname_and_port(): 'databases/database.example.org/foo', ), shell=True, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() @@ -385,7 +422,7 @@ def test_dump_data_sources_runs_pg_dump_with_hostname_and_port(): def test_dump_data_sources_runs_pg_dump_with_username_and_password(): databases = [{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1'}] process = flexmock() - flexmock(module).should_receive('make_extra_environment').and_return( + flexmock(module).should_receive('make_environment').and_return( {'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'} ) flexmock(module).should_receive('make_dump_path').and_return('') @@ -394,6 +431,9 @@ def test_dump_data_sources_runs_pg_dump_with_username_and_password(): 'databases/localhost/foo' ) flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -411,7 +451,7 @@ def test_dump_data_sources_runs_pg_dump_with_username_and_password(): 'databases/localhost/foo', ), shell=True, - extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}, + environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() @@ -428,7 +468,7 @@ def test_dump_data_sources_runs_pg_dump_with_username_and_password(): def test_dump_data_sources_with_username_injection_attack_gets_escaped(): databases = [{'name': 'foo', 'username': 'postgres; naughty-command', 'password': 'trustsome1'}] process = flexmock() - flexmock(module).should_receive('make_extra_environment').and_return( + flexmock(module).should_receive('make_environment').and_return( {'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'} ) flexmock(module).should_receive('make_dump_path').and_return('') @@ -437,6 +477,9 @@ def test_dump_data_sources_with_username_injection_attack_gets_escaped(): 'databases/localhost/foo' ) flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -454,7 +497,7 @@ def test_dump_data_sources_with_username_injection_attack_gets_escaped(): 'databases/localhost/foo', ), shell=True, - extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}, + environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() @@ -470,13 +513,16 @@ def test_dump_data_sources_with_username_injection_attack_gets_escaped(): def test_dump_data_sources_runs_pg_dump_with_directory_format(): databases = [{'name': 'foo', 'format': 'directory'}] - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( 'databases/localhost/foo' ) flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_parent_directory_for_dump') flexmock(module.dump).should_receive('create_named_pipe_for_dump').never() @@ -493,7 +539,7 @@ def test_dump_data_sources_runs_pg_dump_with_directory_format(): 'foo', ), shell=True, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).and_return(flexmock()).once() assert ( @@ -509,16 +555,113 @@ def test_dump_data_sources_runs_pg_dump_with_directory_format(): ) -def test_dump_data_sources_runs_pg_dump_with_options(): - databases = [{'name': 'foo', 'options': '--stuff=such'}] - process = flexmock() - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) +def test_dump_data_sources_runs_pg_dump_with_string_compression(): + databases = [{'name': 'foo', 'compression': 'winrar'}] + processes = [flexmock()] + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( 'databases/localhost/foo' ) flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'pg_dump', + '--no-password', + '--clean', + '--if-exists', + '--format', + 'custom', + '--compress', + 'winrar', + 'foo', + '>', + 'databases/localhost/foo', + ), + shell=True, + environment={'PGSSLMODE': 'disable'}, + run_to_completion=False, + ).and_return(processes[0]).once() + + assert ( + module.dump_data_sources( + databases, + {}, + config_paths=('test.yaml',), + borgmatic_runtime_directory='/run/borgmatic', + patterns=[], + dry_run=False, + ) + == processes + ) + + +def test_dump_data_sources_runs_pg_dump_with_integer_compression(): + databases = [{'name': 'foo', 'compression': 0}] + processes = [flexmock()] + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)) + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( + 'databases/localhost/foo' + ) + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + + flexmock(module).should_receive('execute_command').with_args( + ( + 'pg_dump', + '--no-password', + '--clean', + '--if-exists', + '--format', + 'custom', + '--compress', + '0', + 'foo', + '>', + 'databases/localhost/foo', + ), + shell=True, + environment={'PGSSLMODE': 'disable'}, + run_to_completion=False, + ).and_return(processes[0]).once() + + assert ( + module.dump_data_sources( + databases, + {}, + config_paths=('test.yaml',), + borgmatic_runtime_directory='/run/borgmatic', + patterns=[], + dry_run=False, + ) + == processes + ) + + +def test_dump_data_sources_runs_pg_dump_with_options(): + databases = [{'name': 'foo', 'options': '--stuff=such'}] + process = flexmock() + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_dump_path').and_return('') + flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)) + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( + 'databases/localhost/foo' + ) + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -535,7 +678,7 @@ def test_dump_data_sources_runs_pg_dump_with_options(): 'databases/localhost/foo', ), shell=True, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() @@ -552,19 +695,22 @@ def test_dump_data_sources_runs_pg_dump_with_options(): def test_dump_data_sources_runs_pg_dumpall_for_all_databases(): databases = [{'name': 'all'}] process = flexmock() - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module).should_receive('database_names_to_dump').and_return(('all',)) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( 'databases/localhost/all' ) flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( ('pg_dumpall', '--no-password', '--clean', '--if-exists', '>', 'databases/localhost/all'), shell=True, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() @@ -581,13 +727,16 @@ def test_dump_data_sources_runs_pg_dumpall_for_all_databases(): def test_dump_data_sources_runs_non_default_pg_dump(): databases = [{'name': 'foo', 'pg_dump_command': 'special_pg_dump --compress *'}] process = flexmock() - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path').and_return('') flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)) flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( 'databases/localhost/foo' ) flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.dump).should_receive('create_named_pipe_for_dump') flexmock(module).should_receive('execute_command').with_args( @@ -605,7 +754,7 @@ def test_dump_data_sources_runs_non_default_pg_dump(): 'databases/localhost/foo', ), shell=True, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, run_to_completion=False, ).and_return(process).once() @@ -623,7 +772,10 @@ def test_restore_data_source_dump_runs_pg_restore(): hook_config = [{'name': 'foo', 'schemas': None}, {'name': 'bar'}] extract_process = flexmock(stdout=flexmock()) - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -639,7 +791,7 @@ def test_restore_data_source_dump_runs_pg_restore(): processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( ( @@ -652,7 +804,7 @@ def test_restore_data_source_dump_runs_pg_restore(): '--command', 'ANALYZE', ), - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() module.restore_data_source_dump( @@ -677,7 +829,10 @@ def test_restore_data_source_dump_runs_pg_restore_with_hostname_and_port(): ] extract_process = flexmock(stdout=flexmock()) - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -697,7 +852,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_hostname_and_port(): processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( ( @@ -714,7 +869,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_hostname_and_port(): '--command', 'ANALYZE', ), - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() module.restore_data_source_dump( @@ -739,7 +894,10 @@ def test_restore_data_source_dump_runs_pg_restore_with_username_and_password(): ] extract_process = flexmock(stdout=flexmock()) - flexmock(module).should_receive('make_extra_environment').and_return( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_environment').and_return( {'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'} ) flexmock(module).should_receive('make_dump_path') @@ -759,7 +917,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_username_and_password(): processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}, + environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( ( @@ -774,7 +932,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_username_and_password(): '--command', 'ANALYZE', ), - extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}, + environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}, ).once() module.restore_data_source_dump( @@ -810,7 +968,10 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ ] extract_process = flexmock(stdout=flexmock()) - flexmock(module).should_receive('make_extra_environment').and_return( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_environment').and_return( {'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'} ) flexmock(module).should_receive('make_dump_path') @@ -834,7 +995,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'}, + environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( ( @@ -853,7 +1014,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ '--command', 'ANALYZE', ), - extra_environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'}, + environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'}, ).once() module.restore_data_source_dump( @@ -889,7 +1050,10 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ ] extract_process = flexmock(stdout=flexmock()) - flexmock(module).should_receive('make_extra_environment').and_return( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_environment').and_return( {'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'} ) flexmock(module).should_receive('make_dump_path') @@ -913,7 +1077,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'}, + environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( ( @@ -932,7 +1096,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ '--command', 'ANALYZE', ), - extra_environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'}, + environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'}, ).once() module.restore_data_source_dump( @@ -962,7 +1126,10 @@ def test_restore_data_source_dump_runs_pg_restore_with_options(): ] extract_process = flexmock(stdout=flexmock()) - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -979,7 +1146,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_options(): processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( ( @@ -993,7 +1160,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_options(): '--command', 'ANALYZE', ), - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() module.restore_data_source_dump( @@ -1016,7 +1183,10 @@ def test_restore_data_source_dump_runs_psql_for_all_database_dump(): hook_config = [{'name': 'all', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -1028,11 +1198,11 @@ def test_restore_data_source_dump_runs_psql_for_all_database_dump(): processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( ('psql', '--no-password', '--no-psqlrc', '--quiet', '--command', 'ANALYZE'), - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() module.restore_data_source_dump( @@ -1055,7 +1225,10 @@ def test_restore_data_source_dump_runs_psql_for_plain_database_dump(): hook_config = [{'name': 'foo', 'format': 'plain', 'schemas': None}] extract_process = flexmock(stdout=flexmock()) - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -1063,7 +1236,7 @@ def test_restore_data_source_dump_runs_psql_for_plain_database_dump(): processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( ( @@ -1076,7 +1249,7 @@ def test_restore_data_source_dump_runs_psql_for_plain_database_dump(): '--command', 'ANALYZE', ), - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() module.restore_data_source_dump( @@ -1106,7 +1279,10 @@ def test_restore_data_source_dump_runs_non_default_pg_restore_and_psql(): ] extract_process = flexmock(stdout=flexmock()) - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -1127,7 +1303,7 @@ def test_restore_data_source_dump_runs_non_default_pg_restore_and_psql(): processes=[extract_process], output_log_level=logging.DEBUG, input_file=extract_process.stdout, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( ( @@ -1145,7 +1321,7 @@ def test_restore_data_source_dump_runs_non_default_pg_restore_and_psql(): '--command', 'ANALYZE', ), - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() module.restore_data_source_dump( @@ -1167,7 +1343,10 @@ def test_restore_data_source_dump_runs_non_default_pg_restore_and_psql(): def test_restore_data_source_dump_with_dry_run_skips_restore(): hook_config = [{'name': 'foo', 'schemas': None}] - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename') flexmock(module).should_receive('execute_command_with_processes').never() @@ -1191,7 +1370,10 @@ def test_restore_data_source_dump_with_dry_run_skips_restore(): def test_restore_data_source_dump_without_extract_process_restores_from_disk(): hook_config = [{'name': 'foo', 'schemas': None}] - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('/dump/path') flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -1208,7 +1390,7 @@ def test_restore_data_source_dump_without_extract_process_restores_from_disk(): processes=[], output_log_level=logging.DEBUG, input_file=None, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( ( @@ -1221,7 +1403,7 @@ def test_restore_data_source_dump_without_extract_process_restores_from_disk(): '--command', 'ANALYZE', ), - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() module.restore_data_source_dump( @@ -1243,7 +1425,10 @@ def test_restore_data_source_dump_without_extract_process_restores_from_disk(): def test_restore_data_source_dump_with_schemas_restores_schemas(): hook_config = [{'name': 'foo', 'schemas': ['bar', 'baz']}] - flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'}) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('make_environment').and_return({'PGSSLMODE': 'disable'}) flexmock(module).should_receive('make_dump_path') flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return('/dump/path') flexmock(module).should_receive('execute_command_with_processes').with_args( @@ -1264,7 +1449,7 @@ def test_restore_data_source_dump_with_schemas_restores_schemas(): processes=[], output_log_level=logging.DEBUG, input_file=None, - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() flexmock(module).should_receive('execute_command').with_args( ( @@ -1277,7 +1462,7 @@ def test_restore_data_source_dump_with_schemas_restores_schemas(): '--command', 'ANALYZE', ), - extra_environment={'PGSSLMODE': 'disable'}, + environment={'PGSSLMODE': 'disable'}, ).once() module.restore_data_source_dump( diff --git a/tests/unit/hooks/data_source/test_snapshot.py b/tests/unit/hooks/data_source/test_snapshot.py index 2c23a1e4..8cb98ded 100644 --- a/tests/unit/hooks/data_source/test_snapshot.py +++ b/tests/unit/hooks/data_source/test_snapshot.py @@ -1,34 +1,108 @@ +from flexmock import flexmock + from borgmatic.borg.pattern import Pattern from borgmatic.hooks.data_source import snapshot as module def test_get_contained_patterns_without_candidates_returns_empty(): + flexmock(module.os).should_receive('stat').and_return(flexmock(st_dev=flexmock())) + flexmock(module.os.path).should_receive('exists').and_return(True) + assert module.get_contained_patterns('/mnt', {}) == () def test_get_contained_patterns_with_self_candidate_returns_self(): - candidates = {Pattern('/foo'), Pattern('/mnt'), Pattern('/bar')} + device = flexmock() + flexmock(module.os).should_receive('stat').and_return(flexmock(st_dev=device)) + flexmock(module.os.path).should_receive('exists').and_return(True) + candidates = { + Pattern('/foo', device=device), + Pattern('/mnt', device=device), + Pattern('/bar', device=device), + } - assert module.get_contained_patterns('/mnt', candidates) == (Pattern('/mnt'),) - assert candidates == {Pattern('/foo'), Pattern('/bar')} + assert module.get_contained_patterns('/mnt', candidates) == (Pattern('/mnt', device=device),) + assert candidates == {Pattern('/foo', device=device), Pattern('/bar', device=device)} def test_get_contained_patterns_with_self_candidate_and_caret_prefix_returns_self(): - candidates = {Pattern('^/foo'), Pattern('^/mnt'), Pattern('^/bar')} + device = flexmock() + flexmock(module.os).should_receive('stat').and_return(flexmock(st_dev=device)) + flexmock(module.os.path).should_receive('exists').and_return(True) + candidates = { + Pattern('^/foo', device=device), + Pattern('^/mnt', device=device), + Pattern('^/bar', device=device), + } - assert module.get_contained_patterns('/mnt', candidates) == (Pattern('^/mnt'),) - assert candidates == {Pattern('^/foo'), Pattern('^/bar')} + assert module.get_contained_patterns('/mnt', candidates) == (Pattern('^/mnt', device=device),) + assert candidates == {Pattern('^/foo', device=device), Pattern('^/bar', device=device)} def test_get_contained_patterns_with_child_candidate_returns_child(): - candidates = {Pattern('/foo'), Pattern('/mnt/subdir'), Pattern('/bar')} + device = flexmock() + flexmock(module.os).should_receive('stat').and_return(flexmock(st_dev=device)) + flexmock(module.os.path).should_receive('exists').and_return(True) + candidates = { + Pattern('/foo', device=device), + Pattern('/mnt/subdir', device=device), + Pattern('/bar', device=device), + } - assert module.get_contained_patterns('/mnt', candidates) == (Pattern('/mnt/subdir'),) - assert candidates == {Pattern('/foo'), Pattern('/bar')} + assert module.get_contained_patterns('/mnt', candidates) == ( + Pattern('/mnt/subdir', device=device), + ) + assert candidates == {Pattern('/foo', device=device), Pattern('/bar', device=device)} def test_get_contained_patterns_with_grandchild_candidate_returns_child(): - candidates = {Pattern('/foo'), Pattern('/mnt/sub/dir'), Pattern('/bar')} + device = flexmock() + flexmock(module.os).should_receive('stat').and_return(flexmock(st_dev=device)) + flexmock(module.os.path).should_receive('exists').and_return(True) + candidates = { + Pattern('/foo', device=device), + Pattern('/mnt/sub/dir', device=device), + Pattern('/bar', device=device), + } - assert module.get_contained_patterns('/mnt', candidates) == (Pattern('/mnt/sub/dir'),) - assert candidates == {Pattern('/foo'), Pattern('/bar')} + assert module.get_contained_patterns('/mnt', candidates) == ( + Pattern('/mnt/sub/dir', device=device), + ) + assert candidates == {Pattern('/foo', device=device), Pattern('/bar', device=device)} + + +def test_get_contained_patterns_ignores_child_candidate_on_another_device(): + one_device = flexmock() + another_device = flexmock() + flexmock(module.os).should_receive('stat').and_return(flexmock(st_dev=one_device)) + flexmock(module.os.path).should_receive('exists').and_return(True) + candidates = { + Pattern('/foo', device=one_device), + Pattern('/mnt/subdir', device=another_device), + Pattern('/bar', device=one_device), + } + + assert module.get_contained_patterns('/mnt', candidates) == () + assert candidates == { + Pattern('/foo', device=one_device), + Pattern('/mnt/subdir', device=another_device), + Pattern('/bar', device=one_device), + } + + +def test_get_contained_patterns_with_non_existent_parent_directory_ignores_child_candidate(): + device = flexmock() + flexmock(module.os).should_receive('stat').and_return(flexmock(st_dev=device)) + flexmock(module.os.path).should_receive('exists').and_return(False) + candidates = { + Pattern('/foo', device=device), + Pattern('/mnt/subdir', device=device), + Pattern('/bar', device=device), + } + + assert module.get_contained_patterns('/mnt', candidates) == () + assert candidates == { + Pattern('/foo', device=device), + Pattern('/mnt/subdir', device=device), + Pattern('/bar', device=device), + } diff --git a/tests/unit/hooks/data_source/test_sqlite.py b/tests/unit/hooks/data_source/test_sqlite.py index 03afa3c3..16a4e8f9 100644 --- a/tests/unit/hooks/data_source/test_sqlite.py +++ b/tests/unit/hooks/data_source/test_sqlite.py @@ -107,6 +107,48 @@ def test_dump_data_sources_with_path_injection_attack_gets_escaped(): ) +def test_dump_data_sources_runs_non_default_sqlite_with_path_injection_attack_gets_escaped(): + databases = [ + { + 'path': '/path/to/database1; naughty-command', + 'name': 'database1', + 'sqlite_command': 'custom_sqlite *', + }, + ] + processes = [flexmock()] + + flexmock(module).should_receive('make_dump_path').and_return('/run/borgmatic') + flexmock(module.dump).should_receive('make_data_source_dump_filename').and_return( + '/run/borgmatic/database' + ) + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module.dump).should_receive('create_named_pipe_for_dump') + flexmock(module).should_receive('execute_command').with_args( + ( + 'custom_sqlite', # custom sqlite command + "'*'", # Should get shell escaped to prevent injection attacks. + "'/path/to/database1; naughty-command'", + '.dump', + '>', + '/run/borgmatic/database', + ), + shell=True, + run_to_completion=False, + ).and_return(processes[0]) + + assert ( + module.dump_data_sources( + databases, + {}, + config_paths=('test.yaml',), + borgmatic_runtime_directory='/run/borgmatic', + patterns=[], + dry_run=False, + ) + == processes + ) + + def test_dump_data_sources_with_non_existent_path_warns_and_dumps_database(): databases = [ {'path': '/path/to/database1', 'name': 'database1'}, @@ -216,6 +258,41 @@ def test_restore_data_source_dump_restores_database(): ) +def test_restore_data_source_dump_runs_non_default_sqlite_restores_database(): + hook_config = [ + { + 'path': '/path/to/database', + 'name': 'database', + 'sqlite_restore_command': 'custom_sqlite *', + }, + {'name': 'other'}, + ] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'custom_sqlite', + "'*'", # Should get shell escaped to prevent injection attacks. + '/path/to/database', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + ).once() + + flexmock(module.os).should_receive('remove').once() + + module.restore_data_source_dump( + hook_config, + {}, + data_source=hook_config[0], + dry_run=False, + extract_process=extract_process, + connection_params={'restore_path': None}, + borgmatic_runtime_directory='/run/borgmatic', + ) + + def test_restore_data_source_dump_with_connection_params_uses_connection_params_for_restore(): hook_config = [ {'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'} @@ -245,6 +322,38 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ ) +def test_restore_data_source_dump_runs_non_default_sqlite_with_connection_params_uses_connection_params_for_restore(): + hook_config = [ + {'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'} + ] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'custom_sqlite', + 'cli/path/to/database', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + ).once() + + flexmock(module.os).should_receive('remove').once() + + module.restore_data_source_dump( + hook_config, + {}, + data_source={ + 'name': 'database', + 'sqlite_restore_command': 'custom_sqlite', + }, + dry_run=False, + extract_process=extract_process, + connection_params={'restore_path': 'cli/path/to/database'}, + borgmatic_runtime_directory='/run/borgmatic', + ) + + def test_restore_data_source_dump_without_connection_params_uses_restore_params_in_config_for_restore(): hook_config = [ {'path': '/path/to/database', 'name': 'database', 'restore_path': 'config/path/to/database'} @@ -274,6 +383,40 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ ) +def test_restore_data_source_dump_runs_non_default_sqlite_without_connection_params_uses_restore_params_in_config_for_restore(): + hook_config = [ + { + 'path': '/path/to/database', + 'name': 'database', + 'sqlite_restore_command': 'custom_sqlite', + 'restore_path': 'config/path/to/database', + } + ] + extract_process = flexmock(stdout=flexmock()) + + flexmock(module).should_receive('execute_command_with_processes').with_args( + ( + 'custom_sqlite', + 'config/path/to/database', + ), + processes=[extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + ).once() + + flexmock(module.os).should_receive('remove').once() + + module.restore_data_source_dump( + hook_config, + {}, + data_source=hook_config[0], + dry_run=False, + extract_process=extract_process, + connection_params={'restore_path': None}, + borgmatic_runtime_directory='/run/borgmatic', + ) + + def test_restore_data_source_dump_does_not_restore_database_if_dry_run(): hook_config = [{'path': '/path/to/database', 'name': 'database'}] extract_process = flexmock(stdout=flexmock()) diff --git a/tests/unit/hooks/data_source/test_zfs.py b/tests/unit/hooks/data_source/test_zfs.py index 85782bdb..74ca87b3 100644 --- a/tests/unit/hooks/data_source/test_zfs.py +++ b/tests/unit/hooks/data_source/test_zfs.py @@ -3,7 +3,7 @@ import os import pytest from flexmock import flexmock -from borgmatic.borg.pattern import Pattern, Pattern_style, Pattern_type +from borgmatic.borg.pattern import Pattern, Pattern_source, Pattern_style, Pattern_type from borgmatic.hooks.data_source import zfs as module @@ -11,11 +11,19 @@ def test_get_datasets_to_backup_filters_datasets_by_patterns(): flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).and_return( - 'dataset\t/dataset\t-\nother\t/other\t-', + 'dataset\t/dataset\ton\t-\nother\t/other\ton\t-', ) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' - ).with_args('/dataset', object).and_return((Pattern('/dataset'),)) + ).with_args('/dataset', object).and_return( + ( + Pattern( + '/dataset', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), + ) + ) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' ).with_args('/other', object).and_return(()) @@ -23,24 +31,134 @@ def test_get_datasets_to_backup_filters_datasets_by_patterns(): assert module.get_datasets_to_backup( 'zfs', patterns=( - Pattern('/foo'), - Pattern('/dataset'), - Pattern('/bar'), + Pattern( + '/foo', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), + Pattern( + '/dataset', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), + Pattern( + '/bar', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), ), ) == ( module.Dataset( name='dataset', mount_point='/dataset', - contained_patterns=(Pattern('/dataset'),), + contained_patterns=( + Pattern( + '/dataset', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), + ), ), ) +def test_get_datasets_to_backup_skips_non_root_patterns(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( + 'dataset\t/dataset\ton\t-\nother\t/other\ton\t-', + ) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/dataset', object).and_return( + ( + Pattern( + '/dataset', + Pattern_type.EXCLUDE, + source=Pattern_source.CONFIG, + ), + ) + ) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/other', object).and_return(()) + + assert ( + module.get_datasets_to_backup( + 'zfs', + patterns=( + Pattern( + '/foo', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), + Pattern( + '/dataset', + Pattern_type.EXCLUDE, + source=Pattern_source.CONFIG, + ), + Pattern( + '/bar', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), + ), + ) + == () + ) + + +def test_get_datasets_to_backup_skips_non_config_patterns(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( + 'dataset\t/dataset\ton\t-\nother\t/other\ton\t-', + ) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/dataset', object).and_return( + ( + Pattern( + '/dataset', + Pattern_type.ROOT, + source=Pattern_source.HOOK, + ), + ) + ) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/other', object).and_return(()) + + assert ( + module.get_datasets_to_backup( + 'zfs', + patterns=( + Pattern( + '/foo', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), + Pattern( + '/dataset', + Pattern_type.ROOT, + source=Pattern_source.HOOK, + ), + Pattern( + '/bar', + Pattern_type.ROOT, + source=Pattern_source.CONFIG, + ), + ), + ) + == () + ) + + def test_get_datasets_to_backup_filters_datasets_by_user_property(): flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).and_return( - 'dataset\t/dataset\tauto\nother\t/other\t-', + 'dataset\t/dataset\ton\tauto\nother\t/other\ton\t-', ) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' @@ -49,16 +167,45 @@ def test_get_datasets_to_backup_filters_datasets_by_user_property(): 'get_contained_patterns' ).with_args('/other', object).and_return(()) - assert module.get_datasets_to_backup('zfs', patterns=(Pattern('/foo'), Pattern('/bar'))) == ( + assert module.get_datasets_to_backup( + 'zfs', + patterns=(Pattern('/foo'), Pattern('/bar')), + ) == ( module.Dataset( name='dataset', mount_point='/dataset', auto_backup=True, - contained_patterns=(Pattern('/dataset'),), + contained_patterns=(Pattern('/dataset', source=Pattern_source.HOOK),), ), ) +def test_get_datasets_to_backup_filters_datasets_by_canmount_property(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( + 'dataset\t/dataset\toff\t-\nother\t/other\ton\t-', + ) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/dataset', object).and_return((Pattern('/dataset'),)) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).with_args('/other', object).and_return(()) + + assert ( + module.get_datasets_to_backup( + 'zfs', + patterns=( + Pattern('/foo'), + Pattern('/dataset'), + Pattern('/bar'), + ), + ) + == () + ) + + def test_get_datasets_to_backup_with_invalid_list_output_raises(): flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' @@ -73,11 +220,27 @@ def test_get_datasets_to_backup_with_invalid_list_output_raises(): module.get_datasets_to_backup('zfs', patterns=(Pattern('/foo'), Pattern('/bar'))) -def test_get_all_dataset_mount_points_does_not_filter_datasets(): +def test_get_all_dataset_mount_points_omits_none(): flexmock(module.borgmatic.execute).should_receive( 'execute_command_and_capture_output' ).and_return( - '/dataset\n/other', + '/dataset\nnone\n/other', + ) + flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( + 'get_contained_patterns' + ).and_return((Pattern('/dataset'),)) + + assert module.get_all_dataset_mount_points('zfs') == ( + ('/dataset'), + ('/other'), + ) + + +def test_get_all_dataset_mount_points_omits_duplicates(): + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output' + ).and_return( + '/dataset\n/other\n/dataset\n/other', ) flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive( 'get_contained_patterns' @@ -94,13 +257,13 @@ def test_get_all_dataset_mount_points_does_not_filter_datasets(): ( ( Pattern('/foo/bar/baz'), - Pattern('/run/borgmatic/zfs_snapshots/./foo/bar/baz'), + Pattern('/run/borgmatic/zfs_snapshots/b33f/./foo/bar/baz'), ), - (Pattern('/foo/bar'), Pattern('/run/borgmatic/zfs_snapshots/./foo/bar')), + (Pattern('/foo/bar'), Pattern('/run/borgmatic/zfs_snapshots/b33f/./foo/bar')), ( Pattern('^/foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION), Pattern( - '^/run/borgmatic/zfs_snapshots/./foo/bar', + '^/run/borgmatic/zfs_snapshots/b33f/./foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION, ), @@ -108,46 +271,55 @@ def test_get_all_dataset_mount_points_does_not_filter_datasets(): ( Pattern('/foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION), Pattern( - '/run/borgmatic/zfs_snapshots/./foo/bar', + '/run/borgmatic/zfs_snapshots/b33f/./foo/bar', Pattern_type.INCLUDE, Pattern_style.REGULAR_EXPRESSION, ), ), - (Pattern('/foo'), Pattern('/run/borgmatic/zfs_snapshots/./foo')), - (Pattern('/'), Pattern('/run/borgmatic/zfs_snapshots/./')), + (Pattern('/foo'), Pattern('/run/borgmatic/zfs_snapshots/b33f/./foo')), + (Pattern('/'), Pattern('/run/borgmatic/zfs_snapshots/b33f/./')), ), ) def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path( pattern, expected_pattern ): - assert module.make_borg_snapshot_pattern(pattern, '/run/borgmatic') == expected_pattern + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) + + assert ( + module.make_borg_snapshot_pattern( + pattern, flexmock(mount_point='/something'), '/run/borgmatic' + ) + == expected_pattern + ) def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns(): - flexmock(module).should_receive('get_datasets_to_backup').and_return( - ( - flexmock( - name='dataset', - mount_point='/mnt/dataset', - contained_patterns=(Pattern('/mnt/dataset/subdir'),), - ) - ) + dataset = flexmock( + name='dataset', + mount_point='/mnt/dataset', + contained_patterns=(Pattern('/mnt/dataset/subdir'),), ) + flexmock(module).should_receive('get_datasets_to_backup').and_return((dataset,)) flexmock(module.os).should_receive('getpid').and_return(1234) full_snapshot_name = 'dataset@borgmatic-1234' flexmock(module).should_receive('snapshot_dataset').with_args( 'zfs', full_snapshot_name, ).once() - snapshot_mount_path = '/run/borgmatic/zfs_snapshots/./mnt/dataset' + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) + snapshot_mount_path = '/run/borgmatic/zfs_snapshots/b33f/./mnt/dataset' flexmock(module).should_receive('mount_snapshot').with_args( 'mount', full_snapshot_name, module.os.path.normpath(snapshot_mount_path), ).once() flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/dataset/subdir'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/zfs_snapshots/./mnt/dataset/subdir')) + Pattern('/mnt/dataset/subdir'), dataset, '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/zfs_snapshots/b33f/./mnt/dataset/subdir')) patterns = [Pattern('/mnt/dataset/subdir')] assert ( @@ -188,30 +360,30 @@ def test_dump_data_sources_with_no_datasets_skips_snapshots(): def test_dump_data_sources_uses_custom_commands(): - flexmock(module).should_receive('get_datasets_to_backup').and_return( - ( - flexmock( - name='dataset', - mount_point='/mnt/dataset', - contained_patterns=(Pattern('/mnt/dataset/subdir'),), - ) - ) + dataset = flexmock( + name='dataset', + mount_point='/mnt/dataset', + contained_patterns=(Pattern('/mnt/dataset/subdir'),), ) + flexmock(module).should_receive('get_datasets_to_backup').and_return((dataset,)) flexmock(module.os).should_receive('getpid').and_return(1234) full_snapshot_name = 'dataset@borgmatic-1234' flexmock(module).should_receive('snapshot_dataset').with_args( '/usr/local/bin/zfs', full_snapshot_name, ).once() - snapshot_mount_path = '/run/borgmatic/zfs_snapshots/./mnt/dataset' + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) + snapshot_mount_path = '/run/borgmatic/zfs_snapshots/b33f/./mnt/dataset' flexmock(module).should_receive('mount_snapshot').with_args( '/usr/local/bin/mount', full_snapshot_name, module.os.path.normpath(snapshot_mount_path), ).once() flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/dataset/subdir'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/zfs_snapshots/./mnt/dataset/subdir')) + Pattern('/mnt/dataset/subdir'), dataset, '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/zfs_snapshots/b33f/./mnt/dataset/subdir')) patterns = [Pattern('/mnt/dataset/subdir')] hook_config = { 'zfs_command': '/usr/local/bin/zfs', @@ -261,30 +433,30 @@ def test_dump_data_sources_with_dry_run_skips_commands_and_does_not_touch_patter def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained_patterns(): - flexmock(module).should_receive('get_datasets_to_backup').and_return( - ( - flexmock( - name='dataset', - mount_point='/mnt/dataset', - contained_patterns=(Pattern('/mnt/dataset/subdir'),), - ) - ) + dataset = flexmock( + name='dataset', + mount_point='/mnt/dataset', + contained_patterns=(Pattern('/mnt/dataset/subdir'),), ) + flexmock(module).should_receive('get_datasets_to_backup').and_return((dataset,)) flexmock(module.os).should_receive('getpid').and_return(1234) full_snapshot_name = 'dataset@borgmatic-1234' flexmock(module).should_receive('snapshot_dataset').with_args( 'zfs', full_snapshot_name, ).once() - snapshot_mount_path = '/run/borgmatic/zfs_snapshots/./mnt/dataset' + flexmock(module.hashlib).should_receive('shake_256').and_return( + flexmock(hexdigest=lambda length: 'b33f') + ) + snapshot_mount_path = '/run/borgmatic/zfs_snapshots/b33f/./mnt/dataset' flexmock(module).should_receive('mount_snapshot').with_args( 'mount', full_snapshot_name, module.os.path.normpath(snapshot_mount_path), ).once() flexmock(module).should_receive('make_borg_snapshot_pattern').with_args( - Pattern('/mnt/dataset/subdir'), '/run/borgmatic' - ).and_return(Pattern('/run/borgmatic/zfs_snapshots/./mnt/dataset/subdir')) + Pattern('/mnt/dataset/subdir'), dataset, '/run/borgmatic' + ).and_return(Pattern('/run/borgmatic/zfs_snapshots/b33f/./mnt/dataset/subdir')) patterns = [Pattern('/hmm')] assert ( @@ -317,11 +489,14 @@ def test_remove_data_source_dumps_unmounts_and_destroys_snapshots(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( - 'umount', '/run/borgmatic/zfs_snapshots/mnt/dataset' + 'umount', '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' ).once() flexmock(module).should_receive('get_all_snapshots').and_return( ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid') @@ -343,11 +518,14 @@ def test_remove_data_source_dumps_use_custom_commands(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( - '/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/mnt/dataset' + '/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' ).once() flexmock(module).should_receive('get_all_snapshots').and_return( ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid') @@ -416,11 +594,14 @@ def test_remove_data_source_dumps_bails_for_missing_umount_command(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( - '/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/mnt/dataset' + '/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' ).and_raise(FileNotFoundError) flexmock(module).should_receive('get_all_snapshots').never() flexmock(module).should_receive('destroy_snapshot').never() @@ -434,19 +615,26 @@ def test_remove_data_source_dumps_bails_for_missing_umount_command(): ) -def test_remove_data_source_dumps_bails_for_umount_command_error(): +def test_remove_data_source_dumps_swallows_umount_command_error(): flexmock(module).should_receive('get_all_dataset_mount_points').and_return(('/mnt/dataset',)) flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').with_args( - '/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/mnt/dataset' + '/usr/local/bin/umount', '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' ).and_raise(module.subprocess.CalledProcessError(1, 'wtf')) - flexmock(module).should_receive('get_all_snapshots').never() - flexmock(module).should_receive('destroy_snapshot').never() + flexmock(module).should_receive('get_all_snapshots').and_return( + ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid') + ) + flexmock(module).should_receive('destroy_snapshot').with_args( + '/usr/local/bin/zfs', 'dataset@borgmatic-1234' + ).once() hook_config = {'zfs_command': '/usr/local/bin/zfs', 'umount_command': '/usr/local/bin/umount'} module.remove_data_source_dumps( @@ -462,7 +650,9 @@ def test_remove_data_source_dumps_skips_unmount_snapshot_directories_that_are_no flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(False) flexmock(module.shutil).should_receive('rmtree').never() flexmock(module).should_receive('unmount_snapshot').never() @@ -486,13 +676,50 @@ def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_that_are_no flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/zfs_snapshots' + '/run/borgmatic/zfs_snapshots/b33f' ).and_return(True) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/zfs_snapshots/mnt/dataset' + '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' ).and_return(False) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) + flexmock(module.shutil).should_receive('rmtree') + flexmock(module).should_receive('unmount_snapshot').never() + flexmock(module).should_receive('get_all_snapshots').and_return( + ('dataset@borgmatic-1234', 'dataset@other', 'other@other', 'invalid') + ) + flexmock(module).should_receive('destroy_snapshot').with_args( + 'zfs', 'dataset@borgmatic-1234' + ).once() + + module.remove_data_source_dumps( + hook_config={}, + config={'source_directories': '/mnt/dataset', 'zfs': {}}, + borgmatic_runtime_directory='/run/borgmatic', + dry_run=False, + ) + + +def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_that_are_empty(): + flexmock(module).should_receive('get_all_dataset_mount_points').and_return(('/mnt/dataset',)) + flexmock(module.borgmatic.config.paths).should_receive( + 'replace_temporary_subdirectory_with_glob' + ).and_return('/run/borgmatic') + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) + flexmock(module.os.path).should_receive('isdir').with_args( + '/run/borgmatic/zfs_snapshots/b33f' + ).and_return(True) + flexmock(module.os.path).should_receive('isdir').with_args( + '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' + ).and_return(True) + flexmock(module.os).should_receive('listdir').with_args( + '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' + ).and_return([]) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').never() flexmock(module).should_receive('get_all_snapshots').and_return( @@ -515,13 +742,16 @@ def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_after_rmtre flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/zfs_snapshots' + '/run/borgmatic/zfs_snapshots/b33f' ).and_return(True) flexmock(module.os.path).should_receive('isdir').with_args( - '/run/borgmatic/zfs_snapshots/mnt/dataset' + '/run/borgmatic/zfs_snapshots/b33f/mnt/dataset' ).and_return(True).and_return(False) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree') flexmock(module).should_receive('unmount_snapshot').never() flexmock(module).should_receive('get_all_snapshots').and_return( @@ -544,8 +774,11 @@ def test_remove_data_source_dumps_with_dry_run_skips_unmount_and_destroy(): flexmock(module.borgmatic.config.paths).should_receive( 'replace_temporary_subdirectory_with_glob' ).and_return('/run/borgmatic') - flexmock(module.glob).should_receive('glob').replace_with(lambda path: [path]) + flexmock(module.glob).should_receive('glob').replace_with( + lambda path: [path.replace('*', 'b33f')] + ) flexmock(module.os.path).should_receive('isdir').and_return(True) + flexmock(module.os).should_receive('listdir').and_return(['file.txt']) flexmock(module.shutil).should_receive('rmtree').never() flexmock(module).should_receive('unmount_snapshot').never() flexmock(module).should_receive('get_all_snapshots').and_return( diff --git a/tests/unit/hooks/monitoring/test_ntfy.py b/tests/unit/hooks/monitoring/test_ntfy.py index a5819ec9..d1a8c198 100644 --- a/tests/unit/hooks/monitoring/test_ntfy.py +++ b/tests/unit/hooks/monitoring/test_ntfy.py @@ -36,6 +36,9 @@ def return_default_message_headers(state=Enum): def test_ping_monitor_minimal_config_hits_hosted_ntfy_on_fail(): hook_config = {'topic': topic} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.FAIL), @@ -57,6 +60,9 @@ def test_ping_monitor_with_access_token_hits_hosted_ntfy_on_fail(): 'topic': topic, 'access_token': 'abc123', } + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.FAIL), @@ -80,6 +86,9 @@ def test_ping_monitor_with_username_password_and_access_token_ignores_username_p 'password': 'fakepassword', 'access_token': 'abc123', } + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.FAIL), @@ -103,6 +112,9 @@ def test_ping_monitor_with_username_password_hits_hosted_ntfy_on_fail(): 'username': 'testuser', 'password': 'fakepassword', } + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.FAIL), @@ -121,6 +133,9 @@ def test_ping_monitor_with_username_password_hits_hosted_ntfy_on_fail(): def test_ping_monitor_with_password_but_no_username_warns(): hook_config = {'topic': topic, 'password': 'fakepassword'} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.FAIL), @@ -140,6 +155,9 @@ def test_ping_monitor_with_password_but_no_username_warns(): def test_ping_monitor_with_username_but_no_password_warns(): hook_config = {'topic': topic, 'username': 'testuser'} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.FAIL), @@ -159,6 +177,9 @@ def test_ping_monitor_with_username_but_no_password_warns(): def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_start(): hook_config = {'topic': topic} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').never() module.ping_monitor( @@ -173,6 +194,9 @@ def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_start(): def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_finish(): hook_config = {'topic': topic} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').never() module.ping_monitor( @@ -187,6 +211,9 @@ def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_finish(): def test_ping_monitor_minimal_config_hits_selfhosted_ntfy_on_fail(): hook_config = {'topic': topic, 'server': custom_base_url} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').with_args( f'{custom_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.FAIL), @@ -205,6 +232,9 @@ def test_ping_monitor_minimal_config_hits_selfhosted_ntfy_on_fail(): def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_fail_dry_run(): hook_config = {'topic': topic} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').never() module.ping_monitor( @@ -219,6 +249,9 @@ def test_ping_monitor_minimal_config_does_not_hit_hosted_ntfy_on_fail_dry_run(): def test_ping_monitor_custom_message_hits_hosted_ntfy_on_fail(): hook_config = {'topic': topic, 'fail': custom_message_config} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=custom_message_headers, auth=None ).and_return(flexmock(ok=True)).once() @@ -235,6 +268,9 @@ def test_ping_monitor_custom_message_hits_hosted_ntfy_on_fail(): def test_ping_monitor_custom_state_hits_hosted_ntfy_on_start(): hook_config = {'topic': topic, 'states': ['start', 'fail']} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.START), @@ -253,6 +289,9 @@ def test_ping_monitor_custom_state_hits_hosted_ntfy_on_start(): def test_ping_monitor_with_connection_error_logs_warning(): hook_config = {'topic': topic} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').with_args( f'{default_base_url}/{topic}', headers=return_default_message_headers(borgmatic.hooks.monitoring.monitor.State.FAIL), @@ -270,8 +309,29 @@ def test_ping_monitor_with_connection_error_logs_warning(): ) +def test_ping_monitor_with_credential_error_logs_warning(): + hook_config = {'topic': topic} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).and_raise(ValueError) + flexmock(module.requests).should_receive('post').never() + flexmock(module.logger).should_receive('warning').once() + + module.ping_monitor( + hook_config, + {}, + 'config.yaml', + borgmatic.hooks.monitoring.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, + ) + + def test_ping_monitor_with_other_error_logs_warning(): hook_config = {'topic': topic} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) response = flexmock(ok=False) response.should_receive('raise_for_status').and_raise( module.requests.exceptions.RequestException diff --git a/tests/unit/hooks/monitoring/test_pagerduty.py b/tests/unit/hooks/monitoring/test_pagerduty.py index bf7cc7ba..34c55d95 100644 --- a/tests/unit/hooks/monitoring/test_pagerduty.py +++ b/tests/unit/hooks/monitoring/test_pagerduty.py @@ -3,7 +3,50 @@ from flexmock import flexmock from borgmatic.hooks.monitoring import pagerduty as module +def mock_logger(): + logger = flexmock() + logger.should_receive('addHandler') + logger.should_receive('removeHandler') + flexmock(module.logging).should_receive('getLogger').and_return(logger) + + +def test_initialize_monitor_creates_log_handler(): + monitoring_log_level = 1 + + mock_logger() + flexmock(module.borgmatic.hooks.monitoring.logs).should_receive( + 'Forgetful_buffering_handler' + ).once() + + module.initialize_monitor({}, {}, 'test.yaml', monitoring_log_level, dry_run=False) + + +def test_initialize_monitor_creates_log_handler_when_send_logs_true(): + mock_logger() + flexmock(module.borgmatic.hooks.monitoring.logs).should_receive( + 'Forgetful_buffering_handler' + ).once() + + module.initialize_monitor( + {'send_logs': True}, {}, 'test.yaml', monitoring_log_level=1, dry_run=False + ) + + +def test_initialize_monitor_bails_when_send_logs_false(): + mock_logger() + flexmock(module.borgmatic.hooks.monitoring.logs).should_receive( + 'Forgetful_buffering_handler' + ).never() + + module.initialize_monitor( + {'send_logs': False}, {}, 'test.yaml', monitoring_log_level=1, dry_run=False + ) + + def test_ping_monitor_ignores_start_state(): + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').never() module.ping_monitor( @@ -17,6 +60,9 @@ def test_ping_monitor_ignores_start_state(): def test_ping_monitor_ignores_finish_state(): + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.requests).should_receive('post').never() module.ping_monitor( @@ -30,6 +76,12 @@ def test_ping_monitor_ignores_finish_state(): def test_ping_monitor_calls_api_for_fail_state(): + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.monitoring.logs).should_receive( + 'format_buffered_logs_for_payload' + ).and_return('loggy\nlogs') flexmock(module.requests).should_receive('post').and_return(flexmock(ok=True)) module.ping_monitor( @@ -43,6 +95,12 @@ def test_ping_monitor_calls_api_for_fail_state(): def test_ping_monitor_dry_run_does_not_call_api(): + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.monitoring.logs).should_receive( + 'format_buffered_logs_for_payload' + ).and_return('loggy\nlogs') flexmock(module.requests).should_receive('post').never() module.ping_monitor( @@ -56,6 +114,12 @@ def test_ping_monitor_dry_run_does_not_call_api(): def test_ping_monitor_with_connection_error_logs_warning(): + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.monitoring.logs).should_receive( + 'format_buffered_logs_for_payload' + ).and_return('loggy\nlogs') flexmock(module.requests).should_receive('post').and_raise( module.requests.exceptions.ConnectionError ) @@ -71,8 +135,31 @@ def test_ping_monitor_with_connection_error_logs_warning(): ) +def test_ping_monitor_with_credential_error_logs_warning(): + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).and_raise(ValueError) + flexmock(module.requests).should_receive('post').never() + flexmock(module.logger).should_receive('warning') + + module.ping_monitor( + {'integration_key': 'abc123'}, + {}, + 'config.yaml', + module.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, + ) + + def test_ping_monitor_with_other_error_logs_warning(): response = flexmock(ok=False) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.borgmatic.hooks.monitoring.logs).should_receive( + 'format_buffered_logs_for_payload' + ).and_return('loggy\nlogs') response.should_receive('raise_for_status').and_raise( module.requests.exceptions.RequestException ) diff --git a/tests/unit/hooks/monitoring/test_pushover.py b/tests/unit/hooks/monitoring/test_pushover.py index 281a88f0..be97994b 100644 --- a/tests/unit/hooks/monitoring/test_pushover.py +++ b/tests/unit/hooks/monitoring/test_pushover.py @@ -11,6 +11,9 @@ def test_ping_monitor_config_with_minimum_config_fail_state_backup_successfully_ should be auto populated with the default value which is the state name. ''' hook_config = {'token': 'ksdjfwoweijfvwoeifvjmwghagy92', 'user': '983hfe0of902lkjfa2amanfgui'} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -38,6 +41,9 @@ def test_ping_monitor_config_with_minimum_config_start_state_backup_not_send_to_ 'start' state. Only the 'fail' state is enabled by default. ''' hook_config = {'token': 'ksdjfwoweijfvwoeifvjmwghagy92', 'user': '983hfe0of902lkjfa2amanfgui'} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').never() @@ -63,6 +69,9 @@ def test_ping_monitor_start_state_backup_default_message_successfully_send_to_pu 'user': '983hfe0of902lkjfa2amanfgui', 'states': {'start', 'fail', 'finish'}, } + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -96,6 +105,9 @@ def test_ping_monitor_start_state_backup_custom_message_successfully_send_to_pus 'states': {'start', 'fail', 'finish'}, 'start': {'message': 'custom start message'}, } + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -128,6 +140,9 @@ def test_ping_monitor_start_state_backup_default_message_with_priority_emergency 'states': {'start', 'fail', 'finish'}, 'start': {'priority': 2}, } + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -163,6 +178,9 @@ def test_ping_monitor_start_state_backup_default_message_with_priority_emergency 'states': {'start', 'fail', 'finish'}, 'start': {'priority': 2, 'expire': 600}, } + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -198,6 +216,9 @@ def test_ping_monitor_start_state_backup_default_message_with_priority_emergency 'states': {'start', 'fail', 'finish'}, 'start': {'priority': 2, 'retry': 30}, } + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -236,6 +257,9 @@ def test_ping_monitor_start_state_backup_default_message_with_priority_high_decl 'start': {'priority': 1, 'expire': 30, 'retry': 30}, } + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').never() @@ -288,6 +312,9 @@ def test_ping_monitor_start_state_backup_based_on_documentation_advanced_example 'url_title': 'Login to ticketing system', }, } + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -351,6 +378,9 @@ def test_ping_monitor_fail_state_backup_based_on_documentation_advanced_example_ 'url_title': 'Login to ticketing system', }, } + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -419,6 +449,9 @@ def test_ping_monitor_finish_state_backup_based_on_documentation_advanced_exampl 'url_title': 'Login to ticketing system', }, } + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').with_args( 'https://api.pushover.net/1/messages.json', @@ -446,12 +479,15 @@ def test_ping_monitor_finish_state_backup_based_on_documentation_advanced_exampl ) -def test_ping_monitor_config_with_minimum_config_fail_state_backup_successfully_send_to_pushover_dryrun(): +def test_ping_monitor_config_with_minimum_config_fail_state_backup_successfully_send_to_pushover_dry_run(): ''' This test should be the minimum working configuration. The "message" should be auto populated with the default value which is the state name. ''' hook_config = {'token': 'ksdjfwoweijfvwoeifvjmwghagy92', 'user': '983hfe0of902lkjfa2amanfgui'} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').and_return(flexmock(ok=True)).never() @@ -473,6 +509,9 @@ def test_ping_monitor_config_incorrect_state_exit_early(): 'token': 'ksdjfwoweijfvwoeifvjmwghagy92', 'user': '983hfe0of902lkjfa2amanfgui', } + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').never() flexmock(module.requests).should_receive('post').and_return(flexmock(ok=True)).never() @@ -486,7 +525,7 @@ def test_ping_monitor_config_incorrect_state_exit_early(): ) -def test_ping_monitor_push_post_error_exits_early(): +def test_ping_monitor_push_post_error_bails(): ''' This test simulates the Pushover servers not responding with a 200 OK. We should raise for status and warn then exit. @@ -496,6 +535,9 @@ def test_ping_monitor_push_post_error_exits_early(): 'user': '983hfe0of902lkjfa2amanfgui', } + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) push_response = flexmock(ok=False) push_response.should_receive('raise_for_status').and_raise( module.requests.ConnectionError @@ -520,3 +562,25 @@ def test_ping_monitor_push_post_error_exits_early(): monitoring_log_level=1, dry_run=False, ) + + +def test_ping_monitor_credential_error_bails(): + hook_config = hook_config = { + 'token': 'ksdjfwoweijfvwoeifvjmwghagy92', + 'user': '983hfe0of902lkjfa2amanfgui', + } + + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).and_raise(ValueError) + flexmock(module.requests).should_receive('post').never() + flexmock(module.logger).should_receive('warning').once() + + module.ping_monitor( + hook_config, + {}, + 'config.yaml', + borgmatic.hooks.monitoring.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, + ) diff --git a/tests/unit/hooks/monitoring/test_uptimekuma.py b/tests/unit/hooks/monitoring/test_uptimekuma.py index b29e1c44..a9d00bee 100644 --- a/tests/unit/hooks/monitoring/test_uptimekuma.py +++ b/tests/unit/hooks/monitoring/test_uptimekuma.py @@ -10,7 +10,7 @@ CUSTOM_PUSH_URL = 'https://uptime.example.com/api/push/efgh5678' def test_ping_monitor_hits_default_uptimekuma_on_fail(): hook_config = {} flexmock(module.requests).should_receive('get').with_args( - f'{DEFAULT_PUSH_URL}?status=down&msg=fail' + f'{DEFAULT_PUSH_URL}?status=down&msg=fail', verify=True ).and_return(flexmock(ok=True)).once() module.ping_monitor( @@ -26,7 +26,7 @@ def test_ping_monitor_hits_default_uptimekuma_on_fail(): def test_ping_monitor_hits_custom_uptimekuma_on_fail(): hook_config = {'push_url': CUSTOM_PUSH_URL} flexmock(module.requests).should_receive('get').with_args( - f'{CUSTOM_PUSH_URL}?status=down&msg=fail' + f'{CUSTOM_PUSH_URL}?status=down&msg=fail', verify=True ).and_return(flexmock(ok=True)).once() module.ping_monitor( @@ -42,7 +42,7 @@ def test_ping_monitor_hits_custom_uptimekuma_on_fail(): def test_ping_monitor_custom_uptimekuma_on_start(): hook_config = {'push_url': CUSTOM_PUSH_URL} flexmock(module.requests).should_receive('get').with_args( - f'{CUSTOM_PUSH_URL}?status=up&msg=start' + f'{CUSTOM_PUSH_URL}?status=up&msg=start', verify=True ).and_return(flexmock(ok=True)).once() module.ping_monitor( @@ -58,7 +58,7 @@ def test_ping_monitor_custom_uptimekuma_on_start(): def test_ping_monitor_custom_uptimekuma_on_finish(): hook_config = {'push_url': CUSTOM_PUSH_URL} flexmock(module.requests).should_receive('get').with_args( - f'{CUSTOM_PUSH_URL}?status=up&msg=finish' + f'{CUSTOM_PUSH_URL}?status=up&msg=finish', verify=True ).and_return(flexmock(ok=True)).once() module.ping_monitor( @@ -116,7 +116,7 @@ def test_ping_monitor_does_not_hit_custom_uptimekuma_on_finish_dry_run(): def test_ping_monitor_with_connection_error_logs_warning(): hook_config = {'push_url': CUSTOM_PUSH_URL} flexmock(module.requests).should_receive('get').with_args( - f'{CUSTOM_PUSH_URL}?status=down&msg=fail' + f'{CUSTOM_PUSH_URL}?status=down&msg=fail', verify=True ).and_raise(module.requests.exceptions.ConnectionError) flexmock(module.logger).should_receive('warning').once() @@ -137,7 +137,7 @@ def test_ping_monitor_with_other_error_logs_warning(): module.requests.exceptions.RequestException ) flexmock(module.requests).should_receive('get').with_args( - f'{CUSTOM_PUSH_URL}?status=down&msg=fail' + f'{CUSTOM_PUSH_URL}?status=down&msg=fail', verify=True ).and_return(response) flexmock(module.logger).should_receive('warning').once() @@ -163,3 +163,35 @@ def test_ping_monitor_with_invalid_run_state(): monitoring_log_level=1, dry_run=True, ) + + +def test_ping_monitor_skips_ssl_verification_when_verify_tls_false(): + hook_config = {'push_url': CUSTOM_PUSH_URL, 'verify_tls': False} + flexmock(module.requests).should_receive('get').with_args( + f'{CUSTOM_PUSH_URL}?status=down&msg=fail', verify=False + ).and_return(flexmock(ok=True)).once() + + module.ping_monitor( + hook_config, + {}, + 'config.yaml', + borgmatic.hooks.monitoring.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, + ) + + +def test_ping_monitor_executes_ssl_verification_when_verify_tls_true(): + hook_config = {'push_url': CUSTOM_PUSH_URL, 'verify_tls': True} + flexmock(module.requests).should_receive('get').with_args( + f'{CUSTOM_PUSH_URL}?status=down&msg=fail', verify=True + ).and_return(flexmock(ok=True)).once() + + module.ping_monitor( + hook_config, + {}, + 'config.yaml', + borgmatic.hooks.monitoring.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, + ) diff --git a/tests/unit/hooks/monitoring/test_zabbix.py b/tests/unit/hooks/monitoring/test_zabbix.py index 77193d67..0c2ea577 100644 --- a/tests/unit/hooks/monitoring/test_zabbix.py +++ b/tests/unit/hooks/monitoring/test_zabbix.py @@ -24,7 +24,6 @@ DATA_HOST_KEY_WITH_KEY_VALUE = { 'method': 'history.push', 'params': {'host': HOST, 'key': KEY, 'value': VALUE}, 'id': 1, - 'auth': '3fe6ed01a69ebd79907a120bcd04e494', } DATA_ITEMID = { @@ -39,7 +38,6 @@ DATA_HOST_KEY_WITH_ITEMID = { 'method': 'history.push', 'params': {'itemid': ITEMID, 'value': VALUE}, 'id': 1, - 'auth': '3fe6ed01a69ebd79907a120bcd04e494', } DATA_USER_LOGIN = { @@ -49,17 +47,99 @@ DATA_USER_LOGIN = { 'id': 1, } -AUTH_HEADERS_API_KEY = { +DATA_USER_LOGOUT = { + 'jsonrpc': '2.0', + 'method': 'user.logout', + 'params': [], + 'id': 1, +} + +AUTH_HEADERS_LOGIN = { + 'Content-Type': 'application/json-rpc', +} + +AUTH_HEADERS = { 'Content-Type': 'application/json-rpc', 'Authorization': f'Bearer {API_KEY}', } -AUTH_HEADERS_USERNAME_PASSWORD = {'Content-Type': 'application/json-rpc'} + +def test_send_zabbix_request_with_post_error_bails(): + server = flexmock() + headers = flexmock() + data = {'method': 'do.stuff'} + response = flexmock(ok=False) + response.should_receive('raise_for_status').and_raise( + module.requests.exceptions.RequestException + ) + + flexmock(module.requests).should_receive('post').with_args( + server, headers=headers, json=data + ).and_return(response) + + assert module.send_zabbix_request(server, headers, data) is None -def test_ping_monitor_with_non_matching_state_exits_early(): +def test_send_zabbix_request_with_invalid_json_response_bails(): + server = flexmock() + headers = flexmock() + data = {'method': 'do.stuff'} + flexmock(module.requests.exceptions.JSONDecodeError).should_receive('__init__') + response = flexmock(ok=True) + response.should_receive('json').and_raise(module.requests.exceptions.JSONDecodeError) + + flexmock(module.requests).should_receive('post').with_args( + server, headers=headers, json=data + ).and_return(response) + + assert module.send_zabbix_request(server, headers, data) is None + + +def test_send_zabbix_request_with_success_returns_response_result(): + server = flexmock() + headers = flexmock() + data = {'method': 'do.stuff'} + response = flexmock(ok=True) + response.should_receive('json').and_return({'result': {'foo': 'bar'}}) + + flexmock(module.requests).should_receive('post').with_args( + server, headers=headers, json=data + ).and_return(response) + + assert module.send_zabbix_request(server, headers, data) == {'foo': 'bar'} + + +def test_send_zabbix_request_with_success_passes_through_missing_result(): + server = flexmock() + headers = flexmock() + data = {'method': 'do.stuff'} + response = flexmock(ok=True) + response.should_receive('json').and_return({}) + + flexmock(module.requests).should_receive('post').with_args( + server, headers=headers, json=data + ).and_return(response) + + assert module.send_zabbix_request(server, headers, data) is None + + +def test_send_zabbix_request_with_error_bails(): + server = flexmock() + headers = flexmock() + data = {'method': 'do.stuff'} + response = flexmock(ok=True) + response.should_receive('json').and_return({'result': {'data': [{'error': 'oops'}]}}) + + flexmock(module.requests).should_receive('post').with_args( + server, headers=headers, json=data + ).and_return(response) + + assert module.send_zabbix_request(server, headers, data) is None + + +def test_ping_monitor_with_non_matching_state_bails(): hook_config = {'api_key': API_KEY} - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() module.ping_monitor( hook_config, @@ -71,12 +151,15 @@ def test_ping_monitor_with_non_matching_state_exits_early(): ) -def test_ping_monitor_config_with_api_key_only_exit_early(): +def test_ping_monitor_config_with_api_key_only_bails(): # This test should exit early since only providing an API KEY is not enough # for the hook to work hook_config = {'api_key': API_KEY} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() module.ping_monitor( hook_config, @@ -88,12 +171,15 @@ def test_ping_monitor_config_with_api_key_only_exit_early(): ) -def test_ping_monitor_config_with_host_only_exit_early(): +def test_ping_monitor_config_with_host_only_bails(): # This test should exit early since only providing a HOST is not enough # for the hook to work hook_config = {'host': HOST} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() module.ping_monitor( hook_config, @@ -105,12 +191,15 @@ def test_ping_monitor_config_with_host_only_exit_early(): ) -def test_ping_monitor_config_with_key_only_exit_early(): +def test_ping_monitor_config_with_key_only_bails(): # This test should exit early since only providing a KEY is not enough # for the hook to work hook_config = {'key': KEY} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() module.ping_monitor( hook_config, @@ -122,12 +211,15 @@ def test_ping_monitor_config_with_key_only_exit_early(): ) -def test_ping_monitor_config_with_server_only_exit_early(): +def test_ping_monitor_config_with_server_only_bails(): # This test should exit early since only providing a SERVER is not enough # for the hook to work hook_config = {'server': SERVER} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() module.ping_monitor( hook_config, @@ -139,11 +231,14 @@ def test_ping_monitor_config_with_server_only_exit_early(): ) -def test_ping_monitor_config_user_password_no_zabbix_data_exit_early(): +def test_ping_monitor_config_user_password_no_zabbix_data_bails(): # This test should exit early since there are HOST/KEY or ITEMID provided to publish data to hook_config = {'server': SERVER, 'username': USERNAME, 'password': PASSWORD} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() module.ping_monitor( hook_config, @@ -155,11 +250,14 @@ def test_ping_monitor_config_user_password_no_zabbix_data_exit_early(): ) -def test_ping_monitor_config_api_key_no_zabbix_data_exit_early(): +def test_ping_monitor_config_api_key_no_zabbix_data_bails(): # This test should exit early since there are HOST/KEY or ITEMID provided to publish data to hook_config = {'server': SERVER, 'api_key': API_KEY} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() module.ping_monitor( hook_config, @@ -171,12 +269,15 @@ def test_ping_monitor_config_api_key_no_zabbix_data_exit_early(): ) -def test_ping_monitor_config_itemid_no_auth_data_exit_early(): +def test_ping_monitor_config_itemid_no_auth_data_bails(): # This test should exit early since there is no authentication provided # and Zabbix requires authentication to use it's API hook_config = {'server': SERVER, 'itemid': ITEMID} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() module.ping_monitor( hook_config, @@ -188,12 +289,15 @@ def test_ping_monitor_config_itemid_no_auth_data_exit_early(): ) -def test_ping_monitor_config_host_and_key_no_auth_data_exit_early(): +def test_ping_monitor_config_host_and_key_no_auth_data_bails(): # This test should exit early since there is no authentication provided # and Zabbix requires authentication to use it's API hook_config = {'server': SERVER, 'host': HOST, 'key': KEY} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() module.ping_monitor( hook_config, @@ -209,11 +313,14 @@ def test_ping_monitor_config_host_and_key_with_api_key_auth_data_successful(): # This test should simulate a successful POST to a Zabbix server. This test uses API_KEY # to authenticate and HOST/KEY to know which item to populate in Zabbix. hook_config = {'server': SERVER, 'host': HOST, 'key': KEY, 'api_key': API_KEY} - flexmock(module.requests).should_receive('post').with_args( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('send_zabbix_request').with_args( f'{SERVER}', - headers=AUTH_HEADERS_API_KEY, - json=DATA_HOST_KEY, - ).and_return(flexmock(ok=True)).once() + headers=AUTH_HEADERS, + data=DATA_HOST_KEY, + ).once() flexmock(module.logger).should_receive('warning').never() module.ping_monitor( @@ -226,10 +333,19 @@ def test_ping_monitor_config_host_and_key_with_api_key_auth_data_successful(): ) -def test_ping_monitor_config_host_and_missing_key_exits_early(): - hook_config = {'server': SERVER, 'host': HOST, 'api_key': API_KEY} - flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').never() +def test_ping_monitor_config_adds_missing_api_endpoint_to_server_url(): + # This test should simulate a successful POST to a Zabbix server. This test uses API_KEY + # to authenticate and HOST/KEY to know which item to populate in Zabbix. + hook_config = {'server': SERVER, 'host': HOST, 'key': KEY, 'api_key': API_KEY} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('send_zabbix_request').with_args( + f'{SERVER}', + headers=AUTH_HEADERS, + data=DATA_HOST_KEY, + ).once() + flexmock(module.logger).should_receive('warning').never() module.ping_monitor( hook_config, @@ -241,10 +357,31 @@ def test_ping_monitor_config_host_and_missing_key_exits_early(): ) -def test_ping_monitor_config_key_and_missing_host_exits_early(): - hook_config = {'server': SERVER, 'key': KEY, 'api_key': API_KEY} +def test_ping_monitor_config_host_and_missing_key_bails(): + hook_config = {'server': SERVER, 'host': HOST, 'api_key': API_KEY} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() + + module.ping_monitor( + hook_config, + {}, + 'config.yaml', + borgmatic.hooks.monitoring.monitor.State.FAIL, + monitoring_log_level=1, + dry_run=False, + ) + + +def test_ping_monitor_config_key_and_missing_host_bails(): + hook_config = {'server': SERVER, 'key': KEY, 'api_key': API_KEY} + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module.logger).should_receive('warning').once() + flexmock(module).should_receive('send_zabbix_request').never() module.ping_monitor( hook_config, @@ -267,24 +404,29 @@ def test_ping_monitor_config_host_and_key_with_username_password_auth_data_succe 'password': PASSWORD, } - auth_response = flexmock(ok=True) - auth_response.should_receive('json').and_return( - {'jsonrpc': '2.0', 'result': '3fe6ed01a69ebd79907a120bcd04e494', 'id': 1} - ) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) - flexmock(module.requests).should_receive('post').with_args( + flexmock(module).should_receive('send_zabbix_request').with_args( f'{SERVER}', - headers=AUTH_HEADERS_USERNAME_PASSWORD, - json=DATA_USER_LOGIN, - ).and_return(auth_response).once() + headers=AUTH_HEADERS_LOGIN, + data=DATA_USER_LOGIN, + ).and_return('fakekey').once() flexmock(module.logger).should_receive('warning').never() - flexmock(module.requests).should_receive('post').with_args( + flexmock(module).should_receive('send_zabbix_request').with_args( f'{SERVER}', - headers=AUTH_HEADERS_USERNAME_PASSWORD, - json=DATA_HOST_KEY_WITH_KEY_VALUE, - ).and_return(flexmock(ok=True)).once() + headers=AUTH_HEADERS, + data=DATA_HOST_KEY_WITH_KEY_VALUE, + ).once() + + flexmock(module).should_receive('send_zabbix_request').with_args( + f'{SERVER}', + headers=AUTH_HEADERS, + data=DATA_USER_LOGOUT, + ).once() module.ping_monitor( hook_config, @@ -296,7 +438,7 @@ def test_ping_monitor_config_host_and_key_with_username_password_auth_data_succe ) -def test_ping_monitor_config_host_and_key_with_username_password_auth_data_and_auth_post_error_exits_early(): +def test_ping_monitor_config_host_and_key_with_username_password_auth_data_and_auth_post_error_bails(): hook_config = { 'server': SERVER, 'host': HOST, @@ -305,24 +447,25 @@ def test_ping_monitor_config_host_and_key_with_username_password_auth_data_and_a 'password': PASSWORD, } - auth_response = flexmock(ok=False) - auth_response.should_receive('json').and_return( - {'jsonrpc': '2.0', 'result': '3fe6ed01a69ebd79907a120bcd04e494', 'id': 1} - ) - auth_response.should_receive('raise_for_status').and_raise( - module.requests.ConnectionError - ).once() + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) - flexmock(module.requests).should_receive('post').with_args( + flexmock(module).should_receive('send_zabbix_request').with_args( f'{SERVER}', - headers=AUTH_HEADERS_USERNAME_PASSWORD, - json=DATA_USER_LOGIN, - ).and_return(auth_response).once() - flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').with_args( + headers=AUTH_HEADERS_LOGIN, + data=DATA_USER_LOGIN, + ).and_return(None).once() + flexmock(module).should_receive('send_zabbix_request').with_args( f'{SERVER}', - headers=AUTH_HEADERS_USERNAME_PASSWORD, - json=DATA_HOST_KEY_WITH_KEY_VALUE, + headers=AUTH_HEADERS, + data=DATA_HOST_KEY_WITH_KEY_VALUE, + ).never() + + flexmock(module).should_receive('send_zabbix_request').with_args( + f'{SERVER}', + headers=AUTH_HEADERS, + data=DATA_USER_LOGOUT, ).never() module.ping_monitor( @@ -335,7 +478,7 @@ def test_ping_monitor_config_host_and_key_with_username_password_auth_data_and_a ) -def test_ping_monitor_config_host_and_key_with_username_and_missing_password_exits_early(): +def test_ping_monitor_config_host_and_key_with_username_and_missing_password_bails(): hook_config = { 'server': SERVER, 'host': HOST, @@ -343,8 +486,11 @@ def test_ping_monitor_config_host_and_key_with_username_and_missing_password_exi 'username': USERNAME, } + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() module.ping_monitor( hook_config, @@ -356,7 +502,7 @@ def test_ping_monitor_config_host_and_key_with_username_and_missing_password_exi ) -def test_ping_monitor_config_host_and_key_with_passing_and_missing_username_exits_early(): +def test_ping_monitor_config_host_and_key_with_password_and_missing_username_bails(): hook_config = { 'server': SERVER, 'host': HOST, @@ -364,8 +510,11 @@ def test_ping_monitor_config_host_and_key_with_passing_and_missing_username_exit 'password': PASSWORD, } + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) flexmock(module.logger).should_receive('warning').once() - flexmock(module.requests).should_receive('post').never() + flexmock(module).should_receive('send_zabbix_request').never() module.ping_monitor( hook_config, @@ -381,11 +530,14 @@ def test_ping_monitor_config_itemid_with_api_key_auth_data_successful(): # This test should simulate a successful POST to a Zabbix server. This test uses API_KEY # to authenticate and HOST/KEY to know which item to populate in Zabbix. hook_config = {'server': SERVER, 'itemid': ITEMID, 'api_key': API_KEY} - flexmock(module.requests).should_receive('post').with_args( + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) + flexmock(module).should_receive('send_zabbix_request').with_args( f'{SERVER}', - headers=AUTH_HEADERS_API_KEY, - json=DATA_ITEMID, - ).and_return(flexmock(ok=True)).once() + headers=AUTH_HEADERS, + data=DATA_ITEMID, + ).once() flexmock(module.logger).should_receive('warning').never() module.ping_monitor( @@ -403,24 +555,29 @@ def test_ping_monitor_config_itemid_with_username_password_auth_data_successful( # to authenticate and HOST/KEY to know which item to populate in Zabbix. hook_config = {'server': SERVER, 'itemid': ITEMID, 'username': USERNAME, 'password': PASSWORD} - auth_response = flexmock(ok=True) - auth_response.should_receive('json').and_return( - {'jsonrpc': '2.0', 'result': '3fe6ed01a69ebd79907a120bcd04e494', 'id': 1} - ) + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).replace_with(lambda value, config: value) - flexmock(module.requests).should_receive('post').with_args( + flexmock(module).should_receive('send_zabbix_request').with_args( f'{SERVER}', - headers=AUTH_HEADERS_USERNAME_PASSWORD, - json=DATA_USER_LOGIN, - ).and_return(auth_response).once() + headers=AUTH_HEADERS_LOGIN, + data=DATA_USER_LOGIN, + ).and_return('fakekey').once() flexmock(module.logger).should_receive('warning').never() - flexmock(module.requests).should_receive('post').with_args( + flexmock(module).should_receive('send_zabbix_request').with_args( f'{SERVER}', - headers=AUTH_HEADERS_USERNAME_PASSWORD, - json=DATA_HOST_KEY_WITH_ITEMID, - ).and_return(flexmock(ok=True)).once() + headers=AUTH_HEADERS, + data=DATA_HOST_KEY_WITH_ITEMID, + ).once() + + flexmock(module).should_receive('send_zabbix_request').with_args( + f'{SERVER}', + headers=AUTH_HEADERS, + data=DATA_USER_LOGOUT, + ).once() module.ping_monitor( hook_config, @@ -432,30 +589,13 @@ def test_ping_monitor_config_itemid_with_username_password_auth_data_successful( ) -def test_ping_monitor_config_itemid_with_username_password_auth_data_and_push_post_error_exits_early(): +def test_ping_monitor_with_credential_error_bails(): hook_config = {'server': SERVER, 'itemid': ITEMID, 'username': USERNAME, 'password': PASSWORD} - auth_response = flexmock(ok=True) - auth_response.should_receive('json').and_return( - {'jsonrpc': '2.0', 'result': '3fe6ed01a69ebd79907a120bcd04e494', 'id': 1} - ) - - flexmock(module.requests).should_receive('post').with_args( - f'{SERVER}', - headers=AUTH_HEADERS_USERNAME_PASSWORD, - json=DATA_USER_LOGIN, - ).and_return(auth_response).once() - - push_response = flexmock(ok=False) - push_response.should_receive('raise_for_status').and_raise( - module.requests.ConnectionError - ).once() - flexmock(module.requests).should_receive('post').with_args( - f'{SERVER}', - headers=AUTH_HEADERS_USERNAME_PASSWORD, - json=DATA_HOST_KEY_WITH_ITEMID, - ).and_return(push_response).once() - + flexmock(module.borgmatic.hooks.credential.parse).should_receive( + 'resolve_credential' + ).and_raise(ValueError) + flexmock(module).should_receive('send_zabbix_request').never() flexmock(module.logger).should_receive('warning').once() module.ping_monitor( diff --git a/tests/unit/hooks/test_command.py b/tests/unit/hooks/test_command.py index 7f2e7384..ca6af167 100644 --- a/tests/unit/hooks/test_command.py +++ b/tests/unit/hooks/test_command.py @@ -1,6 +1,7 @@ import logging import subprocess +import pytest from flexmock import flexmock from borgmatic.hooks import command as module @@ -45,46 +46,164 @@ def test_make_environment_with_pyinstaller_clears_LD_LIBRARY_PATH(): def test_make_environment_with_pyinstaller_and_LD_LIBRARY_PATH_ORIG_copies_it_into_LD_LIBRARY_PATH(): assert module.make_environment( {'LD_LIBRARY_PATH_ORIG': '/lib/lib/lib'}, sys_module=flexmock(frozen=True, _MEIPASS='yup') - ) == {'LD_LIBRARY_PATH': '/lib/lib/lib'} + ) == {'LD_LIBRARY_PATH_ORIG': '/lib/lib/lib', 'LD_LIBRARY_PATH': '/lib/lib/lib'} -def test_execute_hook_invokes_each_command(): +@pytest.mark.parametrize( + 'hooks,filters,expected_hooks', + ( + ( + ( + { + 'before': 'action', + 'run': ['foo'], + }, + { + 'after': 'action', + 'run': ['bar'], + }, + { + 'before': 'repository', + 'run': ['baz'], + }, + ), + {}, + ( + { + 'before': 'action', + 'run': ['foo'], + }, + { + 'after': 'action', + 'run': ['bar'], + }, + { + 'before': 'repository', + 'run': ['baz'], + }, + ), + ), + ( + ( + { + 'before': 'action', + 'run': ['foo'], + }, + { + 'after': 'action', + 'run': ['bar'], + }, + { + 'before': 'repository', + 'run': ['baz'], + }, + ), + { + 'before': 'action', + }, + ( + { + 'before': 'action', + 'run': ['foo'], + }, + ), + ), + ( + ( + { + 'after': 'action', + 'run': ['foo'], + }, + { + 'before': 'action', + 'run': ['bar'], + }, + { + 'after': 'repository', + 'run': ['baz'], + }, + ), + { + 'after': 'action', + }, + ( + { + 'after': 'action', + 'run': ['foo'], + }, + ), + ), + ( + ( + { + 'before': 'action', + 'run': ['foo'], + }, + { + 'before': 'action', + 'run': ['bar'], + }, + { + 'before': 'action', + 'run': ['baz'], + }, + ), + { + 'before': 'action', + 'action_names': ['create', 'compact', 'extract'], + }, + ( + { + 'before': 'action', + 'run': ['foo'], + }, + { + 'before': 'action', + 'run': ['bar'], + }, + { + 'before': 'action', + 'run': ['baz'], + }, + ), + ), + ), +) +def test_filter_hooks(hooks, filters, expected_hooks): + assert module.filter_hooks(hooks, **filters) == expected_hooks + + +LOGGING_ANSWER = flexmock() + + +def test_execute_hooks_invokes_each_hook_and_command(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module).should_receive('interpolate_context').replace_with( lambda hook_description, command, context: command ) flexmock(module).should_receive('make_environment').and_return({}) - flexmock(module.borgmatic.execute).should_receive('execute_command').with_args( - [':'], - output_log_level=logging.WARNING, - shell=True, - extra_environment={}, - ).once() - module.execute_hook([':'], None, 'config.yaml', 'pre-backup', dry_run=False) + for command in ('foo', 'bar', 'baz'): + flexmock(module.borgmatic.execute).should_receive('execute_command').with_args( + [command], + output_log_level=LOGGING_ANSWER, + shell=True, + environment={}, + working_directory=None, + ).once() - -def test_execute_hook_with_multiple_commands_invokes_each_command(): - flexmock(module).should_receive('interpolate_context').replace_with( - lambda hook_description, command, context: command + module.execute_hooks( + [{'before': 'create', 'run': ['foo']}, {'before': 'create', 'run': ['bar', 'baz']}], + umask=None, + working_directory=None, + dry_run=False, ) - flexmock(module).should_receive('make_environment').and_return({}) - flexmock(module.borgmatic.execute).should_receive('execute_command').with_args( - [':'], - output_log_level=logging.WARNING, - shell=True, - extra_environment={}, - ).once() - flexmock(module.borgmatic.execute).should_receive('execute_command').with_args( - ['true'], - output_log_level=logging.WARNING, - shell=True, - extra_environment={}, - ).once() - - module.execute_hook([':', 'true'], None, 'config.yaml', 'pre-backup', dry_run=False) -def test_execute_hook_with_umask_sets_that_umask(): +def test_execute_hooks_with_umask_sets_that_umask(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module).should_receive('interpolate_context').replace_with( lambda hook_description, command, context: command ) @@ -92,42 +211,292 @@ def test_execute_hook_with_umask_sets_that_umask(): flexmock(module.os).should_receive('umask').with_args(0o22).once() flexmock(module).should_receive('make_environment').and_return({}) flexmock(module.borgmatic.execute).should_receive('execute_command').with_args( - [':'], - output_log_level=logging.WARNING, + ['foo'], + output_log_level=logging.ANSWER, shell=True, - extra_environment={}, + environment={}, + working_directory=None, ) - module.execute_hook([':'], 77, 'config.yaml', 'pre-backup', dry_run=False) + module.execute_hooks( + [{'before': 'create', 'run': ['foo']}], umask=77, working_directory=None, dry_run=False + ) -def test_execute_hook_with_dry_run_skips_commands(): +def test_execute_hooks_with_working_directory_executes_command_with_it(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER + flexmock(module).should_receive('interpolate_context').replace_with( + lambda hook_description, command, context: command + ) + flexmock(module).should_receive('make_environment').and_return({}) + flexmock(module.borgmatic.execute).should_receive('execute_command').with_args( + ['foo'], + output_log_level=logging.ANSWER, + shell=True, + environment={}, + working_directory='/working', + ) + + module.execute_hooks( + [{'before': 'create', 'run': ['foo']}], + umask=None, + working_directory='/working', + dry_run=False, + ) + + +def test_execute_hooks_with_dry_run_skips_commands(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module).should_receive('interpolate_context').replace_with( lambda hook_description, command, context: command ) flexmock(module).should_receive('make_environment').and_return({}) flexmock(module.borgmatic.execute).should_receive('execute_command').never() - module.execute_hook([':', 'true'], None, 'config.yaml', 'pre-backup', dry_run=True) + module.execute_hooks( + [{'before': 'create', 'run': ['foo']}], umask=None, working_directory=None, dry_run=True + ) -def test_execute_hook_with_empty_commands_does_not_raise(): - module.execute_hook([], None, 'config.yaml', 'post-backup', dry_run=False) +def test_execute_hooks_with_empty_commands_does_not_raise(): + module.execute_hooks([], umask=None, working_directory=None, dry_run=True) -def test_execute_hook_on_error_logs_as_error(): +def test_execute_hooks_with_error_logs_as_error(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER flexmock(module).should_receive('interpolate_context').replace_with( lambda hook_description, command, context: command ) flexmock(module).should_receive('make_environment').and_return({}) flexmock(module.borgmatic.execute).should_receive('execute_command').with_args( - [':'], + ['foo'], output_log_level=logging.ERROR, shell=True, - extra_environment={}, + environment={}, + working_directory=None, ).once() - module.execute_hook([':'], None, 'config.yaml', 'on-error', dry_run=False) + module.execute_hooks( + [{'after': 'error', 'run': ['foo']}], umask=None, working_directory=None, dry_run=False + ) + + +def test_execute_hooks_with_before_or_after_raises(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER + flexmock(module).should_receive('interpolate_context').never() + flexmock(module).should_receive('make_environment').never() + flexmock(module.borgmatic.execute).should_receive('execute_command').never() + + with pytest.raises(ValueError): + module.execute_hooks( + [ + {'erstwhile': 'create', 'run': ['foo']}, + {'erstwhile': 'create', 'run': ['bar', 'baz']}, + ], + umask=None, + working_directory=None, + dry_run=False, + ) + + +def test_execute_hooks_without_commands_to_run_does_not_raise(): + flexmock(module.borgmatic.logger).should_receive('add_custom_log_levels') + flexmock(module.logging).ANSWER = LOGGING_ANSWER + flexmock(module).should_receive('interpolate_context').replace_with( + lambda hook_description, command, context: command + ) + flexmock(module).should_receive('make_environment').and_return({}) + + for command in ('foo', 'bar'): + flexmock(module.borgmatic.execute).should_receive('execute_command').with_args( + [command], + output_log_level=LOGGING_ANSWER, + shell=True, + environment={}, + working_directory=None, + ).once() + + module.execute_hooks( + [{'before': 'create', 'run': []}, {'before': 'create', 'run': ['foo', 'bar']}], + umask=None, + working_directory=None, + dry_run=False, + ) + + +def test_before_after_hooks_calls_command_hooks(): + commands = [ + {'before': 'repository', 'run': ['foo', 'bar']}, + {'after': 'repository', 'run': ['baz']}, + ] + flexmock(module).should_receive('filter_hooks').with_args( + commands, + before='action', + hook_name='myhook', + action_names=['create'], + ).and_return(flexmock()).once() + flexmock(module).should_receive('filter_hooks').with_args( + commands, + after='action', + hook_name='myhook', + action_names=['create'], + ).and_return(flexmock()).once() + flexmock(module).should_receive('execute_hooks').twice() + + with module.Before_after_hooks( + command_hooks=commands, + before_after='action', + umask=1234, + working_directory='/working', + dry_run=False, + hook_name='myhook', + action_names=['create'], + context1='stuff', + context2='such', + ): + pass + + +def test_before_after_hooks_with_before_error_runs_after_hook_and_raises(): + commands = [ + {'before': 'repository', 'run': ['foo', 'bar']}, + {'after': 'repository', 'run': ['baz']}, + ] + flexmock(module).should_receive('filter_hooks').with_args( + commands, + before='action', + hook_name='myhook', + action_names=['create'], + ).and_return(flexmock()).once() + flexmock(module).should_receive('filter_hooks').with_args( + commands, + after='action', + hook_name='myhook', + action_names=['create'], + ).and_return(flexmock()).once() + flexmock(module).should_receive('execute_hooks').and_raise(OSError).and_return(None) + flexmock(module).should_receive('considered_soft_failure').and_return(False) + + with pytest.raises(ValueError): + with module.Before_after_hooks( + command_hooks=commands, + before_after='action', + umask=1234, + working_directory='/working', + dry_run=False, + hook_name='myhook', + action_names=['create'], + context1='stuff', + context2='such', + ): + assert False # This should never get called. + + +def test_before_after_hooks_with_before_soft_failure_does_not_raise(): + commands = [ + {'before': 'repository', 'run': ['foo', 'bar']}, + {'after': 'repository', 'run': ['baz']}, + ] + flexmock(module).should_receive('filter_hooks').with_args( + commands, + before='action', + hook_name='myhook', + action_names=['create'], + ).and_return(flexmock()).once() + flexmock(module).should_receive('filter_hooks').with_args( + commands, + after='action', + hook_name='myhook', + action_names=['create'], + ).and_return(flexmock()).once() + flexmock(module).should_receive('execute_hooks').and_raise(OSError) + flexmock(module).should_receive('considered_soft_failure').and_return(True) + + with module.Before_after_hooks( + command_hooks=commands, + before_after='action', + umask=1234, + working_directory='/working', + dry_run=False, + hook_name='myhook', + action_names=['create'], + context1='stuff', + context2='such', + ): + pass + + +def test_before_after_hooks_with_after_error_raises(): + commands = [ + {'before': 'repository', 'run': ['foo', 'bar']}, + {'after': 'repository', 'run': ['baz']}, + ] + flexmock(module).should_receive('filter_hooks').with_args( + commands, + before='action', + hook_name='myhook', + action_names=['create'], + ).and_return(flexmock()).once() + flexmock(module).should_receive('filter_hooks').with_args( + commands, + after='action', + hook_name='myhook', + action_names=['create'], + ).and_return(flexmock()).once() + flexmock(module).should_receive('execute_hooks').and_return(None).and_raise(OSError) + flexmock(module).should_receive('considered_soft_failure').and_return(False) + + with pytest.raises(ValueError): + with module.Before_after_hooks( + command_hooks=commands, + before_after='action', + umask=1234, + working_directory='/working', + dry_run=False, + hook_name='myhook', + action_names=['create'], + context1='stuff', + context2='such', + ): + pass + + +def test_before_after_hooks_with_after_soft_failure_does_not_raise(): + commands = [ + {'before': 'repository', 'run': ['foo', 'bar']}, + {'after': 'repository', 'run': ['baz']}, + ] + flexmock(module).should_receive('filter_hooks').with_args( + commands, + before='action', + hook_name='myhook', + action_names=['create'], + ).and_return(flexmock()).once() + flexmock(module).should_receive('filter_hooks').with_args( + commands, + after='action', + hook_name='myhook', + action_names=['create'], + ).and_return(flexmock()).once() + flexmock(module).should_receive('execute_hooks').and_return(None).and_raise(OSError) + flexmock(module).should_receive('considered_soft_failure').and_return(True) + + with module.Before_after_hooks( + command_hooks=commands, + before_after='action', + umask=1234, + working_directory='/working', + dry_run=False, + hook_name='myhook', + action_names=['create'], + context1='stuff', + context2='such', + ): + pass def test_considered_soft_failure_treats_soft_fail_exit_code_as_soft_fail(): diff --git a/tests/unit/test_execute.py b/tests/unit/test_execute.py index 8de49912..f3ef9413 100644 --- a/tests/unit/test_execute.py +++ b/tests/unit/test_execute.py @@ -159,8 +159,15 @@ def test_mask_command_secrets_passes_through_other_commands(): ('foo', 'bar'), None, None, - {'DBPASS': 'secret', 'OTHER': 'thing'}, - 'DBPASS=*** OTHER=*** foo bar', + {'UNKNOWN': 'secret', 'OTHER': 'thing'}, + 'foo bar', + ), + ( + ('foo', 'bar'), + None, + None, + {'PGTHING': 'secret', 'BORG_OTHER': 'thing'}, + 'PGTHING=*** BORG_OTHER=*** foo bar', ), ), ) @@ -176,7 +183,6 @@ def test_log_command_logs_command_constructed_from_arguments( def test_execute_command_calls_full_command(): full_command = ['foo', 'bar'] flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, @@ -185,7 +191,7 @@ def test_execute_command_calls_full_command(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -199,7 +205,6 @@ def test_execute_command_calls_full_command_with_output_file(): full_command = ['foo', 'bar'] output_file = flexmock(name='test') flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, @@ -208,7 +213,7 @@ def test_execute_command_calls_full_command_with_output_file(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stderr=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -221,7 +226,6 @@ def test_execute_command_calls_full_command_with_output_file(): def test_execute_command_calls_full_command_without_capturing_output(): full_command = ['foo', 'bar'] flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, @@ -230,7 +234,7 @@ def test_execute_command_calls_full_command_without_capturing_output(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(wait=lambda: 0)).once() flexmock(module).should_receive('interpret_exit_code').and_return(module.Exit_status.SUCCESS) flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) @@ -245,7 +249,6 @@ def test_execute_command_calls_full_command_with_input_file(): full_command = ['foo', 'bar'] input_file = flexmock(name='test') flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=input_file, @@ -254,7 +257,7 @@ def test_execute_command_calls_full_command_with_input_file(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -267,7 +270,6 @@ def test_execute_command_calls_full_command_with_input_file(): def test_execute_command_calls_full_command_with_shell(): full_command = ['foo', 'bar'] flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( ' '.join(full_command), stdin=None, @@ -276,7 +278,7 @@ def test_execute_command_calls_full_command_with_shell(): shell=True, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -286,24 +288,23 @@ def test_execute_command_calls_full_command_with_shell(): assert output is None -def test_execute_command_calls_full_command_with_extra_environment(): +def test_execute_command_calls_full_command_with_environment(): full_command = ['foo', 'bar'] flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, stdout=module.subprocess.PIPE, stderr=module.subprocess.STDOUT, shell=False, - env={'a': 'b', 'c': 'd'}, + env={'a': 'b'}, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') - output = module.execute_command(full_command, extra_environment={'c': 'd'}) + output = module.execute_command(full_command, environment={'a': 'b'}) assert output is None @@ -311,7 +312,6 @@ def test_execute_command_calls_full_command_with_extra_environment(): def test_execute_command_calls_full_command_with_working_directory(): full_command = ['foo', 'bar'] flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, @@ -320,7 +320,7 @@ def test_execute_command_calls_full_command_with_working_directory(): shell=False, env=None, cwd='/working', - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -330,33 +330,10 @@ def test_execute_command_calls_full_command_with_working_directory(): assert output is None -def test_execute_command_with_BORG_PASSPHRASE_FD_leaves_file_descriptors_open(): - full_command = ['foo', 'bar'] - flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) - flexmock(module.subprocess).should_receive('Popen').with_args( - full_command, - stdin=None, - stdout=module.subprocess.PIPE, - stderr=module.subprocess.STDOUT, - shell=False, - env={'a': 'b', 'BORG_PASSPHRASE_FD': '4'}, - cwd=None, - close_fds=False, - ).and_return(flexmock(stdout=None)).once() - flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) - flexmock(module).should_receive('log_outputs') - - output = module.execute_command(full_command, extra_environment={'BORG_PASSPHRASE_FD': '4'}) - - assert output is None - - def test_execute_command_without_run_to_completion_returns_process(): full_command = ['foo', 'bar'] process = flexmock() flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, @@ -365,7 +342,7 @@ def test_execute_command_without_run_to_completion_returns_process(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(process).once() flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') @@ -377,14 +354,14 @@ def test_execute_command_and_capture_output_returns_stdout(): full_command = ['foo', 'bar'] expected_output = '[]' flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('check_output').with_args( full_command, + stdin=None, stderr=None, shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(decode=lambda: expected_output)).once() output = module.execute_command_and_capture_output(full_command) @@ -396,14 +373,14 @@ def test_execute_command_and_capture_output_with_capture_stderr_returns_stderr() full_command = ['foo', 'bar'] expected_output = '[]' flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('check_output').with_args( full_command, + stdin=None, stderr=module.subprocess.STDOUT, shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(decode=lambda: expected_output)).once() output = module.execute_command_and_capture_output(full_command, capture_stderr=True) @@ -416,14 +393,14 @@ def test_execute_command_and_capture_output_returns_output_when_process_error_is expected_output = '[]' err_output = b'[]' flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('check_output').with_args( full_command, + stdin=None, stderr=None, shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_raise(subprocess.CalledProcessError(1, full_command, err_output)).once() flexmock(module).should_receive('interpret_exit_code').and_return( module.Exit_status.SUCCESS @@ -438,14 +415,14 @@ def test_execute_command_and_capture_output_raises_when_command_errors(): full_command = ['foo', 'bar'] expected_output = '[]' flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('check_output').with_args( full_command, + stdin=None, stderr=None, shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_raise(subprocess.CalledProcessError(2, full_command, expected_output)).once() flexmock(module).should_receive('interpret_exit_code').and_return( module.Exit_status.ERROR @@ -459,14 +436,14 @@ def test_execute_command_and_capture_output_returns_output_with_shell(): full_command = ['foo', 'bar'] expected_output = '[]' flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('check_output').with_args( 'foo bar', + stdin=None, stderr=None, shell=True, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(decode=lambda: expected_output)).once() output = module.execute_command_and_capture_output(full_command, shell=True) @@ -474,22 +451,22 @@ def test_execute_command_and_capture_output_returns_output_with_shell(): assert output == expected_output -def test_execute_command_and_capture_output_returns_output_with_extra_environment(): +def test_execute_command_and_capture_output_returns_output_with_environment(): full_command = ['foo', 'bar'] expected_output = '[]' flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('check_output').with_args( full_command, + stdin=None, stderr=None, shell=False, - env={'a': 'b', 'c': 'd'}, + env={'a': 'b'}, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(decode=lambda: expected_output)).once() output = module.execute_command_and_capture_output( - full_command, shell=False, extra_environment={'c': 'd'} + full_command, shell=False, environment={'a': 'b'} ) assert output == expected_output @@ -499,14 +476,14 @@ def test_execute_command_and_capture_output_returns_output_with_working_director full_command = ['foo', 'bar'] expected_output = '[]' flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('check_output').with_args( full_command, + stdin=None, stderr=None, shell=False, env=None, cwd='/working', - close_fds=True, + close_fds=False, ).and_return(flexmock(decode=lambda: expected_output)).once() output = module.execute_command_and_capture_output( @@ -516,34 +493,10 @@ def test_execute_command_and_capture_output_returns_output_with_working_director assert output == expected_output -def test_execute_command_and_capture_output_with_BORG_PASSPHRASE_FD_leaves_file_descriptors_open(): - full_command = ['foo', 'bar'] - expected_output = '[]' - flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) - flexmock(module.subprocess).should_receive('check_output').with_args( - full_command, - stderr=None, - shell=False, - env={'a': 'b', 'BORG_PASSPHRASE_FD': '4'}, - cwd=None, - close_fds=False, - ).and_return(flexmock(decode=lambda: expected_output)).once() - - output = module.execute_command_and_capture_output( - full_command, - shell=False, - extra_environment={'BORG_PASSPHRASE_FD': '4'}, - ) - - assert output == expected_output - - def test_execute_command_with_processes_calls_full_command(): full_command = ['foo', 'bar'] processes = (flexmock(),) flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, @@ -552,8 +505,9 @@ def test_execute_command_with_processes_calls_full_command(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') output = module.execute_command_with_processes(full_command, processes) @@ -565,7 +519,6 @@ def test_execute_command_with_processes_returns_output_with_output_log_level_non full_command = ['foo', 'bar'] processes = (flexmock(),) flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) process = flexmock(stdout=None) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, @@ -575,8 +528,9 @@ def test_execute_command_with_processes_returns_output_with_output_log_level_non shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(process).once() + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs').and_return({process: 'out'}) output = module.execute_command_with_processes(full_command, processes, output_log_level=None) @@ -589,7 +543,6 @@ def test_execute_command_with_processes_calls_full_command_with_output_file(): processes = (flexmock(),) output_file = flexmock(name='test') flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, @@ -598,8 +551,9 @@ def test_execute_command_with_processes_calls_full_command_with_output_file(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stderr=None)).once() + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') output = module.execute_command_with_processes(full_command, processes, output_file=output_file) @@ -611,7 +565,6 @@ def test_execute_command_with_processes_calls_full_command_without_capturing_out full_command = ['foo', 'bar'] processes = (flexmock(),) flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, @@ -620,9 +573,10 @@ def test_execute_command_with_processes_calls_full_command_without_capturing_out shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(wait=lambda: 0)).once() flexmock(module).should_receive('interpret_exit_code').and_return(module.Exit_status.SUCCESS) + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') output = module.execute_command_with_processes( @@ -637,7 +591,6 @@ def test_execute_command_with_processes_calls_full_command_with_input_file(): processes = (flexmock(),) input_file = flexmock(name='test') flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=input_file, @@ -646,8 +599,9 @@ def test_execute_command_with_processes_calls_full_command_with_input_file(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') output = module.execute_command_with_processes(full_command, processes, input_file=input_file) @@ -659,7 +613,6 @@ def test_execute_command_with_processes_calls_full_command_with_shell(): full_command = ['foo', 'bar'] processes = (flexmock(),) flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( ' '.join(full_command), stdin=None, @@ -668,8 +621,9 @@ def test_execute_command_with_processes_calls_full_command_with_shell(): shell=True, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') output = module.execute_command_with_processes(full_command, processes, shell=True) @@ -677,26 +631,24 @@ def test_execute_command_with_processes_calls_full_command_with_shell(): assert output is None -def test_execute_command_with_processes_calls_full_command_with_extra_environment(): +def test_execute_command_with_processes_calls_full_command_with_environment(): full_command = ['foo', 'bar'] processes = (flexmock(),) flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, stdout=module.subprocess.PIPE, stderr=module.subprocess.STDOUT, shell=False, - env={'a': 'b', 'c': 'd'}, + env={'a': 'b'}, cwd=None, - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') - output = module.execute_command_with_processes( - full_command, processes, extra_environment={'c': 'd'} - ) + output = module.execute_command_with_processes(full_command, processes, environment={'a': 'b'}) assert output is None @@ -705,7 +657,6 @@ def test_execute_command_with_processes_calls_full_command_with_working_director full_command = ['foo', 'bar'] processes = (flexmock(),) flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, @@ -714,8 +665,9 @@ def test_execute_command_with_processes_calls_full_command_with_working_director shell=False, env=None, cwd='/working', - close_fds=True, + close_fds=False, ).and_return(flexmock(stdout=None)).once() + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') output = module.execute_command_with_processes( @@ -725,32 +677,6 @@ def test_execute_command_with_processes_calls_full_command_with_working_director assert output is None -def test_execute_command_with_processes_with_BORG_PASSPHRASE_FD_leaves_file_descriptors_open(): - full_command = ['foo', 'bar'] - processes = (flexmock(),) - flexmock(module).should_receive('log_command') - flexmock(module.os, environ={'a': 'b'}) - flexmock(module.subprocess).should_receive('Popen').with_args( - full_command, - stdin=None, - stdout=module.subprocess.PIPE, - stderr=module.subprocess.STDOUT, - shell=False, - env={'a': 'b', 'BORG_PASSPHRASE_FD': '4'}, - cwd=None, - close_fds=False, - ).and_return(flexmock(stdout=None)).once() - flexmock(module).should_receive('log_outputs') - - output = module.execute_command_with_processes( - full_command, - processes, - extra_environment={'BORG_PASSPHRASE_FD': '4'}, - ) - - assert output is None - - def test_execute_command_with_processes_kills_processes_on_error(): full_command = ['foo', 'bar'] flexmock(module).should_receive('log_command') @@ -758,7 +684,6 @@ def test_execute_command_with_processes_kills_processes_on_error(): process.should_receive('poll') process.should_receive('kill').once() processes = (process,) - flexmock(module.os, environ={'a': 'b'}) flexmock(module.subprocess).should_receive('Popen').with_args( full_command, stdin=None, @@ -767,8 +692,9 @@ def test_execute_command_with_processes_kills_processes_on_error(): shell=False, env=None, cwd=None, - close_fds=True, + close_fds=False, ).and_raise(subprocess.CalledProcessError(1, full_command, 'error')).once() + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs').never() with pytest.raises(subprocess.CalledProcessError): diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index a1222862..a43a1497 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -44,19 +44,23 @@ def test_interactive_console_true_when_isatty_and_TERM_is_not_dumb(capsys): assert module.interactive_console() is True -def test_should_do_markup_respects_no_color_value(): - flexmock(module.os.environ).should_receive('get').and_return(None) +def test_should_do_markup_respects_json_enabled_value(): + flexmock(module.os.environ).should_receive('get').never() flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=True, configs={}) is False + assert module.should_do_markup(configs={}, json_enabled=True) is False def test_should_do_markup_respects_config_value(): flexmock(module.os.environ).should_receive('get').and_return(None) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=False, configs={'foo.yaml': {'color': False}}) is False + assert ( + module.should_do_markup(configs={'foo.yaml': {'color': False}}, json_enabled=False) is False + ) flexmock(module).should_receive('interactive_console').and_return(True).once() - assert module.should_do_markup(no_color=False, configs={'foo.yaml': {'color': True}}) is True + assert ( + module.should_do_markup(configs={'foo.yaml': {'color': True}}, json_enabled=False) is True + ) def test_should_do_markup_prefers_any_false_config_value(): @@ -65,11 +69,11 @@ def test_should_do_markup_prefers_any_false_config_value(): assert ( module.should_do_markup( - no_color=False, configs={ 'foo.yaml': {'color': True}, 'bar.yaml': {'color': False}, }, + json_enabled=False, ) is False ) @@ -83,14 +87,16 @@ def test_should_do_markup_respects_PY_COLORS_environment_variable(): flexmock(module).should_receive('to_bool').and_return(True) - assert module.should_do_markup(no_color=False, configs={}) is True + assert module.should_do_markup(configs={}, json_enabled=False) is True -def test_should_do_markup_prefers_no_color_value_to_config_value(): +def test_should_do_markup_prefers_json_enabled_value_to_config_value(): flexmock(module.os.environ).should_receive('get').and_return(None) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=True, configs={'foo.yaml': {'color': True}}) is False + assert ( + module.should_do_markup(configs={'foo.yaml': {'color': True}}, json_enabled=True) is False + ) def test_should_do_markup_prefers_config_value_to_environment_variables(): @@ -98,7 +104,9 @@ def test_should_do_markup_prefers_config_value_to_environment_variables(): flexmock(module).should_receive('to_bool').and_return(True) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=False, configs={'foo.yaml': {'color': False}}) is False + assert ( + module.should_do_markup(configs={'foo.yaml': {'color': False}}, json_enabled=False) is False + ) def test_should_do_markup_prefers_no_color_value_to_environment_variables(): @@ -106,14 +114,14 @@ def test_should_do_markup_prefers_no_color_value_to_environment_variables(): flexmock(module).should_receive('to_bool').and_return(True) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=True, configs={}) is False + assert module.should_do_markup(configs={}, json_enabled=False) is False def test_should_do_markup_respects_interactive_console_value(): flexmock(module.os.environ).should_receive('get').and_return(None) flexmock(module).should_receive('interactive_console').and_return(True) - assert module.should_do_markup(no_color=False, configs={}) is True + assert module.should_do_markup(configs={}, json_enabled=False) is True def test_should_do_markup_prefers_PY_COLORS_to_interactive_console_value(): @@ -124,7 +132,7 @@ def test_should_do_markup_prefers_PY_COLORS_to_interactive_console_value(): flexmock(module).should_receive('to_bool').and_return(True) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=False, configs={}) is True + assert module.should_do_markup(configs={}, json_enabled=False) is True def test_should_do_markup_prefers_NO_COLOR_to_interactive_console_value(): @@ -132,7 +140,7 @@ def test_should_do_markup_prefers_NO_COLOR_to_interactive_console_value(): flexmock(module.os.environ).should_receive('get').with_args('NO_COLOR', None).and_return('True') flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=False, configs={}) is False + assert module.should_do_markup(configs={}, json_enabled=False) is False def test_should_do_markup_respects_NO_COLOR_environment_variable(): @@ -140,7 +148,7 @@ def test_should_do_markup_respects_NO_COLOR_environment_variable(): flexmock(module.os.environ).should_receive('get').with_args('PY_COLORS', None).and_return(None) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=False, configs={}) is False + assert module.should_do_markup(configs={}, json_enabled=False) is False def test_should_do_markup_ignores_empty_NO_COLOR_environment_variable(): @@ -148,7 +156,7 @@ def test_should_do_markup_ignores_empty_NO_COLOR_environment_variable(): flexmock(module.os.environ).should_receive('get').with_args('PY_COLORS', None).and_return(None) flexmock(module).should_receive('interactive_console').and_return(True) - assert module.should_do_markup(no_color=False, configs={}) is True + assert module.should_do_markup(configs={}, json_enabled=False) is True def test_should_do_markup_prefers_NO_COLOR_to_PY_COLORS(): @@ -160,7 +168,7 @@ def test_should_do_markup_prefers_NO_COLOR_to_PY_COLORS(): ) flexmock(module).should_receive('interactive_console').never() - assert module.should_do_markup(no_color=False, configs={}) is False + assert module.should_do_markup(configs={}, json_enabled=False) is False def test_multi_stream_handler_logs_to_handler_for_log_level(): @@ -382,8 +390,8 @@ def test_delayed_logging_handler_flush_forwards_each_record_to_each_target(): handler = module.Delayed_logging_handler() flexmock(handler).should_receive('acquire') flexmock(handler).should_receive('release') - handler.targets = [flexmock(), flexmock()] - handler.buffer = [flexmock(), flexmock()] + handler.targets = [flexmock(level=logging.DEBUG), flexmock(level=logging.DEBUG)] + handler.buffer = [flexmock(levelno=logging.DEBUG), flexmock(levelno=logging.DEBUG)] handler.targets[0].should_receive('handle').with_args(handler.buffer[0]).once() handler.targets[1].should_receive('handle').with_args(handler.buffer[0]).once() handler.targets[0].should_receive('handle').with_args(handler.buffer[1]).once() @@ -394,6 +402,25 @@ def test_delayed_logging_handler_flush_forwards_each_record_to_each_target(): assert handler.buffer == [] +<<<<<<< HEAD +======= +def test_delayed_logging_handler_flush_skips_forwarding_when_log_record_is_too_low_for_target(): + handler = module.Delayed_logging_handler() + flexmock(handler).should_receive('acquire') + flexmock(handler).should_receive('release') + handler.targets = [flexmock(level=logging.INFO), flexmock(level=logging.DEBUG)] + handler.buffer = [flexmock(levelno=logging.DEBUG), flexmock(levelno=logging.INFO)] + handler.targets[0].should_receive('handle').with_args(handler.buffer[0]).never() + handler.targets[1].should_receive('handle').with_args(handler.buffer[0]).once() + handler.targets[0].should_receive('handle').with_args(handler.buffer[1]).once() + handler.targets[1].should_receive('handle').with_args(handler.buffer[1]).once() + + handler.flush() + + assert handler.buffer == [] + + +>>>>>>> main def test_flush_delayed_logging_without_handlers_does_not_raise(): root_logger = flexmock(handlers=[]) root_logger.should_receive('removeHandler') diff --git a/tests/unit/test_signals.py b/tests/unit/test_signals.py index 86bb759c..57ee14ab 100644 --- a/tests/unit/test_signals.py +++ b/tests/unit/test_signals.py @@ -26,7 +26,7 @@ def test_handle_signal_bails_on_recursion(): def test_handle_signal_exits_on_sigterm(): signal_number = module.signal.SIGTERM frame = flexmock(f_back=flexmock(f_code=flexmock(co_name='something'))) - flexmock(module.os).should_receive('getpgrp').and_return(flexmock) + flexmock(module.os).should_receive('getpgrp').and_return(flexmock()) flexmock(module.os).should_receive('killpg') flexmock(module.sys).should_receive('exit').with_args( module.EXIT_CODE_FROM_SIGNAL + signal_number @@ -38,8 +38,10 @@ def test_handle_signal_exits_on_sigterm(): def test_handle_signal_raises_on_sigint(): signal_number = module.signal.SIGINT frame = flexmock(f_back=flexmock(f_code=flexmock(co_name='something'))) - flexmock(module.os).should_receive('getpgrp').and_return(flexmock) - flexmock(module.os).should_receive('killpg') + process_group = flexmock() + flexmock(module.os).should_receive('getpgrp').and_return(process_group) + flexmock(module.os).should_receive('killpg').with_args(process_group, module.signal.SIGINT) + flexmock(module.os).should_receive('killpg').with_args(process_group, module.signal.SIGTERM) flexmock(module.sys).should_receive('exit').never() with pytest.raises(KeyboardInterrupt):