From cc7e01be68d75a55841493f6d1c50312462ee262 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 25 Jan 2025 14:01:25 -0800 Subject: [PATCH 01/12] Log the repository path or label on every relevant log message, not just some logs (#635). --- NEWS | 3 + borgmatic/actions/borg.py | 2 +- borgmatic/actions/break_lock.py | 2 +- borgmatic/actions/change_passphrase.py | 2 +- borgmatic/actions/check.py | 37 ++--- borgmatic/actions/compact.py | 5 +- borgmatic/actions/config/bootstrap.py | 1 - borgmatic/actions/create.py | 8 +- borgmatic/actions/delete.py | 2 +- borgmatic/actions/export_key.py | 2 +- borgmatic/actions/export_tar.py | 2 +- borgmatic/actions/extract.py | 2 +- borgmatic/actions/info.py | 2 +- borgmatic/actions/list.py | 4 +- borgmatic/actions/mount.py | 4 +- borgmatic/actions/prune.py | 2 +- borgmatic/actions/repo_create.py | 2 +- borgmatic/actions/repo_delete.py | 2 +- borgmatic/actions/repo_info.py | 2 +- borgmatic/actions/repo_list.py | 2 +- borgmatic/actions/restore.py | 20 +-- borgmatic/actions/transfer.py | 2 +- borgmatic/borg/create.py | 11 +- borgmatic/commands/borgmatic.py | 148 +++++++++--------- borgmatic/config/paths.py | 15 +- borgmatic/hooks/command.py | 18 +-- borgmatic/hooks/data_source/bootstrap.py | 28 ++-- borgmatic/hooks/data_source/btrfs.py | 44 +++--- borgmatic/hooks/data_source/dump.py | 4 +- borgmatic/hooks/data_source/lvm.py | 53 +++---- borgmatic/hooks/data_source/mariadb.py | 56 +++---- borgmatic/hooks/data_source/mongodb.py | 41 +++-- borgmatic/hooks/data_source/mysql.py | 56 +++---- borgmatic/hooks/data_source/postgresql.py | 50 +++--- borgmatic/hooks/data_source/sqlite.py | 46 +++--- borgmatic/hooks/data_source/zfs.py | 46 +++--- borgmatic/hooks/dispatch.py | 37 +++-- borgmatic/logger.py | 23 ++- pyproject.toml | 2 +- tests/unit/actions/test_check.py | 8 - tests/unit/actions/test_restore.py | 16 -- tests/unit/borg/test_create.py | 4 +- .../unit/hooks/data_source/test_bootstrap.py | 7 - tests/unit/hooks/data_source/test_btrfs.py | 15 -- tests/unit/hooks/data_source/test_lvm.py | 19 --- tests/unit/hooks/data_source/test_mariadb.py | 25 +-- tests/unit/hooks/data_source/test_mongodb.py | 4 +- tests/unit/hooks/data_source/test_mysql.py | 25 +-- .../unit/hooks/data_source/test_postgresql.py | 4 +- tests/unit/hooks/data_source/test_sqlite.py | 4 +- tests/unit/hooks/data_source/test_zfs.py | 16 -- tests/unit/hooks/test_command.py | 24 +-- tests/unit/hooks/test_dispatch.py | 2 +- 53 files changed, 403 insertions(+), 558 deletions(-) diff --git a/NEWS b/NEWS index d2947fbc..47da36b6 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +1.9.9.dev0 + * #635: Log the repository path or label on every relevant log message, not just some logs. + 1.9.8 * #979: Fix root patterns so they don't have an invalid "sh:" prefix before getting passed to Borg. * Expand the recent contributors documentation section to include ticket submitters—not just code diff --git a/borgmatic/actions/borg.py b/borgmatic/actions/borg.py index dc714700..fe2facdb 100644 --- a/borgmatic/actions/borg.py +++ b/borgmatic/actions/borg.py @@ -23,7 +23,7 @@ def run_borg( repository, borg_arguments.repository ): logger.info( - f'{repository.get("label", repository["path"])}: Running arbitrary Borg command' + 'Running arbitrary Borg command' ) archive_name = borgmatic.borg.repo_list.resolve_archive_name( repository['path'], diff --git a/borgmatic/actions/break_lock.py b/borgmatic/actions/break_lock.py index e94ab466..8375a6eb 100644 --- a/borgmatic/actions/break_lock.py +++ b/borgmatic/actions/break_lock.py @@ -22,7 +22,7 @@ def run_break_lock( repository, break_lock_arguments.repository ): logger.info( - f'{repository.get("label", repository["path"])}: Breaking repository and cache locks' + 'Breaking repository and cache locks' ) borgmatic.borg.break_lock.break_lock( repository['path'], diff --git a/borgmatic/actions/change_passphrase.py b/borgmatic/actions/change_passphrase.py index 2a3df3a5..3671d72c 100644 --- a/borgmatic/actions/change_passphrase.py +++ b/borgmatic/actions/change_passphrase.py @@ -25,7 +25,7 @@ def run_change_passphrase( ) ): logger.info( - f'{repository.get("label", repository["path"])}: Changing repository passphrase' + 'Changing repository passphrase' ) borgmatic.borg.change_passphrase.change_passphrase( repository['path'], diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index 8c9575e4..2555613f 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -363,7 +363,6 @@ def collect_spot_check_source_paths( borgmatic.hooks.dispatch.call_hooks( 'use_streaming', config, - repository['path'], borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE, ).values() ) @@ -468,15 +467,14 @@ def compare_spot_check_hashes( global_arguments, local_path, remote_path, - log_prefix, source_paths, ): ''' Given a repository configuration dict, the name of the latest archive, a configuration dict, the local Borg version, global arguments as an argparse.Namespace instance, the local Borg path, the - remote Borg path, a log label, and spot check source paths, compare the hashes for a sampling of - the source paths with hashes from corresponding paths in the given archive. Return a sequence of - the paths that fail that hash comparison. + remote Borg path, and spot check source paths, compare the hashes for a sampling of the source + paths with hashes from corresponding paths in the given archive. Return a sequence of the paths + that fail that hash comparison. ''' # Based on the configured sample percentage, come up with a list of random sample files from the # source directories. @@ -492,7 +490,7 @@ def compare_spot_check_hashes( if os.path.exists(os.path.join(working_directory or '', source_path)) } logger.debug( - f'{log_prefix}: Sampling {sample_count} source paths (~{spot_check_config["data_sample_percentage"]}%) for spot check' + f'Sampling {sample_count} source paths (~{spot_check_config["data_sample_percentage"]}%) for spot check' ) source_sample_paths_iterator = iter(source_sample_paths) @@ -580,8 +578,7 @@ def spot_check( disk to those stored in the latest archive. If any differences are beyond configured tolerances, then the check fails. ''' - log_prefix = f'{repository.get("label", repository["path"])}' - logger.debug(f'{log_prefix}: Running spot check') + logger.debug('Running spot check') try: spot_check_config = next( @@ -604,7 +601,7 @@ def spot_check( remote_path, borgmatic_runtime_directory, ) - logger.debug(f'{log_prefix}: {len(source_paths)} total source paths for spot check') + logger.debug(f'{len(source_paths)} total source paths for spot check') archive = borgmatic.borg.repo_list.resolve_archive_name( repository['path'], @@ -615,7 +612,7 @@ def spot_check( local_path, remote_path, ) - logger.debug(f'{log_prefix}: Using archive {archive} for spot check') + logger.debug(f'Using archive {archive} for spot check') archive_paths = collect_spot_check_archive_paths( repository, @@ -627,11 +624,11 @@ def spot_check( remote_path, borgmatic_runtime_directory, ) - logger.debug(f'{log_prefix}: {len(archive_paths)} total archive paths for spot check') + logger.debug(f'{len(archive_paths)} total archive paths for spot check') if len(source_paths) == 0: logger.debug( - f'{log_prefix}: Paths in latest archive but not source paths: {", ".join(set(archive_paths)) or "none"}' + f'Paths in latest archive but not source paths: {", ".join(set(archive_paths)) or "none"}' ) raise ValueError( 'Spot check failed: There are no source paths to compare against the archive' @@ -644,10 +641,10 @@ def spot_check( if count_delta_percentage > spot_check_config['count_tolerance_percentage']: rootless_source_paths = set(path.lstrip(os.path.sep) for path in source_paths) logger.debug( - f'{log_prefix}: Paths in source paths but not latest archive: {", ".join(rootless_source_paths - set(archive_paths)) or "none"}' + f'Paths in source paths but not latest archive: {", ".join(rootless_source_paths - set(archive_paths)) or "none"}' ) logger.debug( - f'{log_prefix}: Paths in latest archive but not source paths: {", ".join(set(archive_paths) - rootless_source_paths) or "none"}' + f'Paths in latest archive but not source paths: {", ".join(set(archive_paths) - rootless_source_paths) or "none"}' ) raise ValueError( f'Spot check failed: {count_delta_percentage:.2f}% file count delta between source paths and latest archive (tolerance is {spot_check_config["count_tolerance_percentage"]}%)' @@ -661,25 +658,24 @@ def spot_check( global_arguments, local_path, remote_path, - log_prefix, source_paths, ) # Error if the percentage of failing hashes exceeds the configured tolerance percentage. - logger.debug(f'{log_prefix}: {len(failing_paths)} non-matching spot check hashes') + logger.debug(f'{len(failing_paths)} non-matching spot check hashes') data_tolerance_percentage = spot_check_config['data_tolerance_percentage'] failing_percentage = (len(failing_paths) / len(source_paths)) * 100 if failing_percentage > data_tolerance_percentage: logger.debug( - f'{log_prefix}: Source paths with data not matching the latest archive: {", ".join(failing_paths)}' + f'Source paths with data not matching the latest archive: {", ".join(failing_paths)}' ) raise ValueError( f'Spot check failed: {failing_percentage:.2f}% of source paths with data not matching the latest archive (tolerance is {data_tolerance_percentage}%)' ) logger.info( - f'{log_prefix}: Spot check passed with a {count_delta_percentage:.2f}% file count delta and a {failing_percentage:.2f}% file data delta' + f'Spot check passed with a {count_delta_percentage:.2f}% file count delta and a {failing_percentage:.2f}% file data delta' ) @@ -713,8 +709,7 @@ def run_check( **hook_context, ) - log_prefix = repository.get('label', repository['path']) - logger.info(f'{log_prefix}: Running consistency checks') + logger.info(f'Running consistency checks') repository_id = borgmatic.borg.check.get_repository_id( repository['path'], @@ -768,7 +763,7 @@ def run_check( if 'spot' in checks: with borgmatic.config.paths.Runtime_directory( - config, log_prefix + config ) as borgmatic_runtime_directory: spot_check( repository, diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index 29cf8943..970e5d49 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -38,7 +38,7 @@ def run_compact( ) if borgmatic.borg.feature.available(borgmatic.borg.feature.Feature.COMPACT, local_borg_version): logger.info( - f'{repository.get("label", repository["path"])}: Compacting segments{dry_run_label}' + f'Compacting segments{dry_run_label}' ) borgmatic.borg.compact.compact_segments( global_arguments.dry_run, @@ -54,8 +54,9 @@ def run_compact( ) else: # pragma: nocover logger.info( - f'{repository.get("label", repository["path"])}: Skipping compact (only available/needed in Borg 1.2+)' + 'Skipping compact (only available/needed in Borg 1.2+)' ) + borgmatic.hooks.command.execute_hook( config.get('after_compact'), config.get('umask'), diff --git a/borgmatic/actions/config/bootstrap.py b/borgmatic/actions/config/bootstrap.py index cf6b56d6..4a72b2cd 100644 --- a/borgmatic/actions/config/bootstrap.py +++ b/borgmatic/actions/config/bootstrap.py @@ -45,7 +45,6 @@ def get_config_paths(archive_name, bootstrap_arguments, global_arguments, local_ # still want to support reading the manifest from previously created archives as well. with borgmatic.config.paths.Runtime_directory( {'user_runtime_directory': bootstrap_arguments.user_runtime_directory}, - bootstrap_arguments.repository, ) as borgmatic_runtime_directory: for base_directory in ( 'borgmatic', diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index ae231892..1710351c 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -283,17 +283,15 @@ def run_create( **hook_context, ) - log_prefix = repository.get('label', repository['path']) - logger.info(f'{log_prefix}: Creating archive{dry_run_label}') + logger.info(f'Creating archive{dry_run_label}') working_directory = borgmatic.config.paths.get_working_directory(config) with borgmatic.config.paths.Runtime_directory( - config, log_prefix + config ) as borgmatic_runtime_directory: borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_data_source_dumps', config, - repository['path'], borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE, borgmatic_runtime_directory, global_arguments.dry_run, @@ -302,7 +300,6 @@ def run_create( active_dumps = borgmatic.hooks.dispatch.call_hooks( 'dump_data_sources', config, - repository['path'], borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE, config_paths, borgmatic_runtime_directory, @@ -339,7 +336,6 @@ def run_create( borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_data_source_dumps', config, - config_filename, borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE, borgmatic_runtime_directory, global_arguments.dry_run, diff --git a/borgmatic/actions/delete.py b/borgmatic/actions/delete.py index 4ecba019..d4ce5602 100644 --- a/borgmatic/actions/delete.py +++ b/borgmatic/actions/delete.py @@ -23,7 +23,7 @@ def run_delete( if delete_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, delete_arguments.repository ): - logger.answer(f'{repository.get("label", repository["path"])}: Deleting archives') + logger.answer('Deleting archives') archive_name = ( borgmatic.borg.repo_list.resolve_archive_name( diff --git a/borgmatic/actions/export_key.py b/borgmatic/actions/export_key.py index 56a72e7f..067ff176 100644 --- a/borgmatic/actions/export_key.py +++ b/borgmatic/actions/export_key.py @@ -21,7 +21,7 @@ def run_export_key( if export_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, export_arguments.repository ): - logger.info(f'{repository.get("label", repository["path"])}: Exporting repository key') + logger.info('Exporting repository key') borgmatic.borg.export_key.export_key( repository['path'], config, diff --git a/borgmatic/actions/export_tar.py b/borgmatic/actions/export_tar.py index 5f9f2a24..7ce6203e 100644 --- a/borgmatic/actions/export_tar.py +++ b/borgmatic/actions/export_tar.py @@ -23,7 +23,7 @@ def run_export_tar( repository, export_tar_arguments.repository ): logger.info( - f'{repository["path"]}: Exporting archive {export_tar_arguments.archive} as tar file' + f'Exporting archive {export_tar_arguments.archive} as tar file' ) borgmatic.borg.export_tar.export_tar_archive( global_arguments.dry_run, diff --git a/borgmatic/actions/extract.py b/borgmatic/actions/extract.py index d298ae56..fd20a05b 100644 --- a/borgmatic/actions/extract.py +++ b/borgmatic/actions/extract.py @@ -34,7 +34,7 @@ def run_extract( repository, extract_arguments.repository ): logger.info( - f'{repository.get("label", repository["path"])}: Extracting archive {extract_arguments.archive}' + f'Extracting archive {extract_arguments.archive}' ) borgmatic.borg.extract.extract_archive( global_arguments.dry_run, diff --git a/borgmatic/actions/info.py b/borgmatic/actions/info.py index 132ebe5b..76bd7554 100644 --- a/borgmatic/actions/info.py +++ b/borgmatic/actions/info.py @@ -28,7 +28,7 @@ def run_info( ): if not info_arguments.json: logger.answer( - f'{repository.get("label", repository["path"])}: Displaying archive summary information' + 'Displaying archive summary information' ) archive_name = borgmatic.borg.repo_list.resolve_archive_name( repository['path'], diff --git a/borgmatic/actions/list.py b/borgmatic/actions/list.py index c4985eef..0def72fb 100644 --- a/borgmatic/actions/list.py +++ b/borgmatic/actions/list.py @@ -27,9 +27,9 @@ def run_list( ): if not list_arguments.json: if list_arguments.find_paths: # pragma: no cover - logger.answer(f'{repository.get("label", repository["path"])}: Searching archives') + logger.answer('Searching archives') elif not list_arguments.archive: # pragma: no cover - logger.answer(f'{repository.get("label", repository["path"])}: Listing archives') + logger.answer('Listing archives') archive_name = borgmatic.borg.repo_list.resolve_archive_name( repository['path'], diff --git a/borgmatic/actions/mount.py b/borgmatic/actions/mount.py index e50993fe..3bddc619 100644 --- a/borgmatic/actions/mount.py +++ b/borgmatic/actions/mount.py @@ -24,10 +24,10 @@ def run_mount( ): if mount_arguments.archive: logger.info( - f'{repository.get("label", repository["path"])}: Mounting archive {mount_arguments.archive}' + f'Mounting archive {mount_arguments.archive}' ) else: # pragma: nocover - logger.info(f'{repository.get("label", repository["path"])}: Mounting repository') + logger.info('Mounting repository') borgmatic.borg.mount.mount_archive( repository['path'], diff --git a/borgmatic/actions/prune.py b/borgmatic/actions/prune.py index 0cb074b5..83dfe39f 100644 --- a/borgmatic/actions/prune.py +++ b/borgmatic/actions/prune.py @@ -35,7 +35,7 @@ def run_prune( global_arguments.dry_run, **hook_context, ) - logger.info(f'{repository.get("label", repository["path"])}: Pruning archives{dry_run_label}') + logger.info(f'Pruning archives{dry_run_label}') borgmatic.borg.prune.prune_archives( global_arguments.dry_run, repository['path'], diff --git a/borgmatic/actions/repo_create.py b/borgmatic/actions/repo_create.py index 399efdb6..e6252e4e 100644 --- a/borgmatic/actions/repo_create.py +++ b/borgmatic/actions/repo_create.py @@ -23,7 +23,7 @@ def run_repo_create( ): return - logger.info(f'{repository.get("label", repository["path"])}: Creating repository') + logger.info('Creating repository') borgmatic.borg.repo_create.create_repository( global_arguments.dry_run, repository['path'], diff --git a/borgmatic/actions/repo_delete.py b/borgmatic/actions/repo_delete.py index 78d21e5d..242ffaa9 100644 --- a/borgmatic/actions/repo_delete.py +++ b/borgmatic/actions/repo_delete.py @@ -21,7 +21,7 @@ def run_repo_delete( repository, repo_delete_arguments.repository ): logger.answer( - f'{repository.get("label", repository["path"])}: Deleting repository' + 'Deleting repository' + (' cache' if repo_delete_arguments.cache_only else '') ) diff --git a/borgmatic/actions/repo_info.py b/borgmatic/actions/repo_info.py index eef49774..4d4509fa 100644 --- a/borgmatic/actions/repo_info.py +++ b/borgmatic/actions/repo_info.py @@ -26,7 +26,7 @@ def run_repo_info( ): if not repo_info_arguments.json: logger.answer( - f'{repository.get("label", repository["path"])}: Displaying repository summary information' + 'Displaying repository summary information' ) json_output = borgmatic.borg.repo_info.display_repository_info( diff --git a/borgmatic/actions/repo_list.py b/borgmatic/actions/repo_list.py index dfe9a850..39128333 100644 --- a/borgmatic/actions/repo_list.py +++ b/borgmatic/actions/repo_list.py @@ -25,7 +25,7 @@ def run_repo_list( repository, repo_list_arguments.repository ): if not repo_list_arguments.json: - logger.answer(f'{repository.get("label", repository["path"])}: Listing repository') + logger.answer('Listing repository') json_output = borgmatic.borg.repo_list.list_repository( repository['path'], diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index 05eeec58..660e7130 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -71,10 +71,10 @@ def render_dump_metadata(dump): return metadata -def get_configured_data_source(config, restore_dump, log_prefix): +def get_configured_data_source(config, restore_dump): ''' Search in the given configuration dict for dumps corresponding to the given dump to restore. If - there are multiple matches, error. Log using the given log prefix. + there are multiple matches, error. Return the found data source as a data source configuration dict or None if not found. ''' @@ -91,7 +91,6 @@ def get_configured_data_source(config, restore_dump, log_prefix): borgmatic.hooks.dispatch.call_hook( function_name='get_default_port', config=config, - log_prefix=log_prefix, hook_name=hook_name, ), ) @@ -174,13 +173,12 @@ def restore_single_dump( ) logger.info( - f'{repository.get("label", repository["path"])}: Restoring data source {dump_metadata}' + f'Restoring data source {dump_metadata}' ) dump_patterns = borgmatic.hooks.dispatch.call_hooks( 'make_data_source_dump_patterns', config, - repository['path'], borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE, borgmatic_runtime_directory, data_source['name'], @@ -227,7 +225,6 @@ def restore_single_dump( borgmatic.hooks.dispatch.call_hook( function_name='restore_data_source_dump', config=config, - log_prefix=repository['path'], hook_name=hook_name, data_source=data_source, dry_run=global_arguments.dry_run, @@ -319,7 +316,7 @@ def collect_dumps_from_archive( break else: logger.warning( - f'{repository}: Ignoring invalid data source dump path "{dump_path}" in archive {archive}' + f'Ignoring invalid data source dump path "{dump_path}" in archive {archive}' ) return dumps_from_archive @@ -444,16 +441,14 @@ def run_restore( ): return - log_prefix = repository.get('label', repository['path']) - logger.info(f'{log_prefix}: Restoring data sources from archive {restore_arguments.archive}') + logger.info(f'Restoring data sources from archive {restore_arguments.archive}') with borgmatic.config.paths.Runtime_directory( - config, log_prefix + config ) as borgmatic_runtime_directory: borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_data_source_dumps', config, - repository['path'], borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE, borgmatic_runtime_directory, global_arguments.dry_run, @@ -494,7 +489,6 @@ def run_restore( found_data_source = get_configured_data_source( config, restore_dump, - log_prefix=repository['path'], ) # For a dump that wasn't found via an exact match in the configuration, try to fallback @@ -503,7 +497,6 @@ def run_restore( found_data_source = get_configured_data_source( config, Dump(restore_dump.hook_name, 'all', restore_dump.hostname, restore_dump.port), - log_prefix=repository['path'], ) if not found_data_source: @@ -531,7 +524,6 @@ def run_restore( borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_data_source_dumps', config, - repository['path'], borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE, borgmatic_runtime_directory, global_arguments.dry_run, diff --git a/borgmatic/actions/transfer.py b/borgmatic/actions/transfer.py index 4051b14e..ff94b56e 100644 --- a/borgmatic/actions/transfer.py +++ b/borgmatic/actions/transfer.py @@ -18,7 +18,7 @@ def run_transfer( Run the "transfer" action for the given repository. ''' logger.info( - f'{repository.get("label", repository["path"])}: Transferring archives to repository' + 'Transferring archives to repository' ) borgmatic.borg.transfer.transfer_archives( global_arguments.dry_run, diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index c0f4399c..1fb5284d 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -20,14 +20,12 @@ from borgmatic.execute import ( logger = logging.getLogger(__name__) -def write_patterns_file(patterns, borgmatic_runtime_directory, log_prefix, patterns_file=None): +def write_patterns_file(patterns, borgmatic_runtime_directory, patterns_file=None): ''' Given a sequence of patterns as borgmatic.borg.pattern.Pattern instances, write them to a named temporary file in the given borgmatic runtime directory and return the file object so it can continue to exist on disk as long as the caller needs it. - Use the given log prefix in any logging. - If an optional open pattern file is given, append to it instead of making a new temporary file. Return None if no patterns are provided. ''' @@ -43,7 +41,7 @@ def write_patterns_file(patterns, borgmatic_runtime_directory, log_prefix, patte f'{pattern.type.value} {pattern.style.value}{":" if pattern.style.value else ""}{pattern.path}' for pattern in patterns ) - logger.debug(f'{log_prefix}: Writing patterns to {patterns_file.name}:\n{patterns_output}') + logger.debug(f'Writing patterns to {patterns_file.name}:\n{patterns_output}') patterns_file.write(patterns_output) patterns_file.flush() @@ -217,9 +215,7 @@ def make_base_create_command( if config.get('source_directories_must_exist', False): check_all_root_patterns_exist(patterns) - patterns_file = write_patterns_file( - patterns, borgmatic_runtime_directory, log_prefix=repository_path - ) + patterns_file = write_patterns_file(patterns, borgmatic_runtime_directory) checkpoint_interval = config.get('checkpoint_interval', None) checkpoint_volume = config.get('checkpoint_volume', None) chunker_params = config.get('chunker_params', None) @@ -334,7 +330,6 @@ def make_base_create_command( for special_file_path in special_file_paths ), borgmatic_runtime_directory, - log_prefix=repository_path, patterns_file=patterns_file, ) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 5f7331ef..fefbd8e9 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -39,7 +39,7 @@ from borgmatic.commands.arguments import parse_arguments from borgmatic.config import checks, collect, validate from borgmatic.hooks import command, dispatch from borgmatic.hooks.monitoring import monitor -from borgmatic.logger import DISABLED, add_custom_log_levels, configure_logging, should_do_markup +from borgmatic.logger import DISABLED, add_custom_log_levels, configure_logging, should_do_markup, set_log_prefix from borgmatic.signals import configure_signals from borgmatic.verbosity import verbosity_to_log_level @@ -86,12 +86,12 @@ def run_configuration(config_filename, config, config_paths, arguments): if skip_actions: logger.debug( - f"{config_filename}: Skipping {'/'.join(skip_actions)} action{'s' if len(skip_actions) > 1 else ''} due to configured skip_actions" + f"Skipping {'/'.join(skip_actions)} action{'s' if len(skip_actions) > 1 else ''} due to configured skip_actions" ) try: local_borg_version = borg_version.local_borg_version(config, local_path) - logger.debug(f'{config_filename}: Borg {local_borg_version}') + logger.debug(f'Borg {local_borg_version}') except (OSError, CalledProcessError, ValueError) as error: yield from log_error_records(f'{config_filename}: Error getting local Borg version', error) return @@ -101,7 +101,6 @@ def run_configuration(config_filename, config, config_paths, arguments): dispatch.call_hooks( 'initialize_monitor', config, - config_filename, dispatch.Hook_type.MONITORING, monitoring_log_level, global_arguments.dry_run, @@ -110,14 +109,13 @@ def run_configuration(config_filename, config, config_paths, arguments): dispatch.call_hooks( 'ping_monitor', config, - config_filename, dispatch.Hook_type.MONITORING, monitor.State.START, monitoring_log_level, global_arguments.dry_run, ) except (OSError, CalledProcessError) as error: - if command.considered_soft_failure(config_filename, error): + if command.considered_soft_failure(error): return encountered_error = error @@ -130,55 +128,59 @@ def run_configuration(config_filename, config, config_paths, arguments): (repo, 0), ) - while not repo_queue.empty(): - repository, retry_num = repo_queue.get() - logger.debug( - f'{repository.get("label", repository["path"])}: Running actions for repository' - ) - timeout = retry_num * retry_wait - if timeout: - logger.warning( - f'{repository.get("label", repository["path"])}: Sleeping {timeout}s before next retry' + try: + while not repo_queue.empty(): + repository, retry_num = repo_queue.get() + set_log_prefix(repository.get('label', repository['path'])) + logger.debug( + 'Running actions for repository' ) - time.sleep(timeout) - try: - yield from run_actions( - arguments=arguments, - config_filename=config_filename, - config=config, - config_paths=config_paths, - local_path=local_path, - remote_path=remote_path, - local_borg_version=local_borg_version, - repository=repository, - ) - except (OSError, CalledProcessError, ValueError) as error: - if retry_num < retries: - repo_queue.put( - (repository, retry_num + 1), - ) - tuple( # Consume the generator so as to trigger logging. - log_error_records( - f'{repository.get("label", repository["path"])}: Error running actions for repository', - error, - levelno=logging.WARNING, - log_command_error_output=True, - ) - ) + timeout = retry_num * retry_wait + if timeout: logger.warning( - f'{repository.get("label", repository["path"])}: Retrying... attempt {retry_num + 1}/{retries}' + f'Sleeping {timeout}s before next retry' ) - continue + time.sleep(timeout) + try: + yield from run_actions( + arguments=arguments, + config_filename=config_filename, + config=config, + config_paths=config_paths, + local_path=local_path, + remote_path=remote_path, + local_borg_version=local_borg_version, + repository=repository, + ) + except (OSError, CalledProcessError, ValueError) as error: + if retry_num < retries: + repo_queue.put( + (repository, retry_num + 1), + ) + tuple( # Consume the generator so as to trigger logging. + log_error_records( + f'{repository.get("label", repository["path"])}: Error running actions for repository', + error, + levelno=logging.WARNING, + log_command_error_output=True, + ) + ) + logger.warning( + f'Retrying... attempt {retry_num + 1}/{retries}' + ) + continue - if command.considered_soft_failure(config_filename, error): - continue + if command.considered_soft_failure(error): + continue - yield from log_error_records( - f'{repository.get("label", repository["path"])}: Error running actions for repository', - error, - ) - encountered_error = error - error_repository = repository['path'] + yield from log_error_records( + f'Error running actions for repository', + error, + ) + encountered_error = error + error_repository = repository['path'] + finally: + set_log_prefix(config_filename) try: if monitoring_hooks_are_activated: @@ -186,14 +188,13 @@ def run_configuration(config_filename, config, config_paths, arguments): dispatch.call_hooks( 'ping_monitor', config, - config_filename, dispatch.Hook_type.MONITORING, monitor.State.LOG, monitoring_log_level, global_arguments.dry_run, ) except (OSError, CalledProcessError) as error: - if not command.considered_soft_failure(config_filename, error): + if not command.considered_soft_failure(error): encountered_error = error yield from log_error_records(f'{repository["path"]}: Error pinging monitor', error) @@ -203,7 +204,6 @@ def run_configuration(config_filename, config, config_paths, arguments): dispatch.call_hooks( 'ping_monitor', config, - config_filename, dispatch.Hook_type.MONITORING, monitor.State.FINISH, monitoring_log_level, @@ -212,13 +212,12 @@ def run_configuration(config_filename, config, config_paths, arguments): dispatch.call_hooks( 'destroy_monitor', config, - config_filename, dispatch.Hook_type.MONITORING, monitoring_log_level, global_arguments.dry_run, ) except (OSError, CalledProcessError) as error: - if command.considered_soft_failure(config_filename, error): + if command.considered_soft_failure(error): return encountered_error = error @@ -239,7 +238,6 @@ def run_configuration(config_filename, config, config_paths, arguments): dispatch.call_hooks( 'ping_monitor', config, - config_filename, dispatch.Hook_type.MONITORING, monitor.State.FAIL, monitoring_log_level, @@ -248,13 +246,12 @@ def run_configuration(config_filename, config, config_paths, arguments): dispatch.call_hooks( 'destroy_monitor', config, - config_filename, dispatch.Hook_type.MONITORING, monitoring_log_level, global_arguments.dry_run, ) except (OSError, CalledProcessError) as error: - if command.considered_soft_failure(config_filename, error): + if command.considered_soft_failure(error): return yield from log_error_records(f'{config_filename}: Error running on-error hook', error) @@ -819,23 +816,28 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): # Execute the actions corresponding to each configuration file. json_results = [] - for config_filename, config in configs.items(): - results = list(run_configuration(config_filename, config, config_paths, arguments)) - error_logs = tuple(result for result in results if isinstance(result, logging.LogRecord)) - if error_logs: - yield from log_error_records(f'{config_filename}: An error occurred') - yield from error_logs - else: - yield logging.makeLogRecord( - dict( - levelno=logging.INFO, - levelname='INFO', - msg=f'{config_filename}: Successfully ran configuration file', + try: + for config_filename, config in configs.items(): + set_log_prefix(config_filename) + results = list(run_configuration(config_filename, config, config_paths, arguments)) + error_logs = tuple(result for result in results if isinstance(result, logging.LogRecord)) + + if error_logs: + yield from log_error_records('An error occurred') + yield from error_logs + else: + yield logging.makeLogRecord( + dict( + levelno=logging.INFO, + levelname='INFO', + msg='Successfully ran configuration file', + ) ) - ) - if results: - json_results.extend(results) + if results: + json_results.extend(results) + finally: + set_log_prefix(None) if 'umount' in arguments: logger.info(f"Unmounting mount point {arguments['umount'].mount_point}") diff --git a/borgmatic/config/paths.py b/borgmatic/config/paths.py index e5786237..7b13d306 100644 --- a/borgmatic/config/paths.py +++ b/borgmatic/config/paths.py @@ -76,14 +76,13 @@ class Runtime_directory: automatically gets cleaned up as necessary. ''' - def __init__(self, config, log_prefix): + def __init__(self, config): ''' - Given a configuration dict and a log prefix, determine the borgmatic runtime directory, - creating a secure, temporary directory within it if necessary. Defaults to - $XDG_RUNTIME_DIR/./borgmatic or $RUNTIME_DIRECTORY/./borgmatic or - $TMPDIR/borgmatic-[random]/./borgmatic or $TEMP/borgmatic-[random]/./borgmatic or - /tmp/borgmatic-[random]/./borgmatic where "[random]" is a randomly generated string intended - to avoid path collisions. + Given a configuration dict determine the borgmatic runtime directory, creating a secure, + temporary directory within it if necessary. Defaults to $XDG_RUNTIME_DIR/./borgmatic or + $RUNTIME_DIRECTORY/./borgmatic or $TMPDIR/borgmatic-[random]/./borgmatic or + $TEMP/borgmatic-[random]/./borgmatic or /tmp/borgmatic-[random]/./borgmatic where "[random]" + is a randomly generated string intended to avoid path collisions. If XDG_RUNTIME_DIR or RUNTIME_DIRECTORY is set and already ends in "/borgmatic", then don't tack on a second "/borgmatic" path component. @@ -127,7 +126,7 @@ class Runtime_directory: ) os.makedirs(self.runtime_path, mode=0o700, exist_ok=True) - logger.debug(f'{log_prefix}: Using runtime directory {os.path.normpath(self.runtime_path)}') + logger.debug(f'Using runtime directory {os.path.normpath(self.runtime_path)}') def __enter__(self): ''' diff --git a/borgmatic/hooks/command.py b/borgmatic/hooks/command.py index d03fc1da..d0e55b62 100644 --- a/borgmatic/hooks/command.py +++ b/borgmatic/hooks/command.py @@ -12,7 +12,7 @@ logger = logging.getLogger(__name__) SOFT_FAIL_EXIT_CODE = 75 -def interpolate_context(config_filename, hook_description, command, context): +def interpolate_context(hook_description, command, context): ''' Given a config filename, a hook description, a single hook command, and a dict of context names/values, interpolate the values by "{name}" into the command and return the result. @@ -22,7 +22,7 @@ def interpolate_context(config_filename, hook_description, command, context): for unsupported_variable in re.findall(r'{\w+}', command): logger.warning( - f"{config_filename}: Variable '{unsupported_variable}' is not supported in {hook_description} hook" + f"Variable '{unsupported_variable}' is not supported in {hook_description} hook" ) return command @@ -54,26 +54,26 @@ def execute_hook(commands, umask, config_filename, description, dry_run, **conte Raise subprocesses.CalledProcessError if an error occurs in a hook. ''' if not commands: - logger.debug(f'{config_filename}: No commands to run for {description} hook') + logger.debug(f'No commands to run for {description} hook') return dry_run_label = ' (dry run; not actually running hooks)' if dry_run else '' context['configuration_filename'] = config_filename commands = [ - interpolate_context(config_filename, description, command, context) for command in commands + interpolate_context(description, command, context) for command in commands ] if len(commands) == 1: - logger.info(f'{config_filename}: Running command for {description} hook{dry_run_label}') + logger.info(f'Running command for {description} hook{dry_run_label}') else: logger.info( - f'{config_filename}: Running {len(commands)} commands for {description} hook{dry_run_label}', + f'Running {len(commands)} commands for {description} hook{dry_run_label}', ) if umask: parsed_umask = int(str(umask), 8) - logger.debug(f'{config_filename}: Set hook umask to {oct(parsed_umask)}') + logger.debug(f'Set hook umask to {oct(parsed_umask)}') original_umask = os.umask(parsed_umask) else: original_umask = None @@ -94,7 +94,7 @@ def execute_hook(commands, umask, config_filename, description, dry_run, **conte os.umask(original_umask) -def considered_soft_failure(config_filename, error): +def considered_soft_failure(error): ''' Given a configuration filename and an exception object, return whether the exception object represents a subprocess.CalledProcessError with a return code of SOFT_FAIL_EXIT_CODE. If so, @@ -106,7 +106,7 @@ def considered_soft_failure(config_filename, error): if exit_code == SOFT_FAIL_EXIT_CODE: logger.info( - f'{config_filename}: Command hook exited with soft failure exit code ({SOFT_FAIL_EXIT_CODE}); skipping remaining repository actions', + f'Command hook exited with soft failure exit code ({SOFT_FAIL_EXIT_CODE}); skipping remaining repository actions', ) return True diff --git a/borgmatic/hooks/data_source/bootstrap.py b/borgmatic/hooks/data_source/bootstrap.py index 360ccc47..b6ac1b4d 100644 --- a/borgmatic/hooks/data_source/bootstrap.py +++ b/borgmatic/hooks/data_source/bootstrap.py @@ -10,7 +10,7 @@ import borgmatic.config.paths logger = logging.getLogger(__name__) -def use_streaming(hook_config, config, log_prefix): # pragma: no cover +def use_streaming(hook_config, config): # pragma: no cover ''' Return whether dump streaming is used for this hook. (Spoiler: It isn't.) ''' @@ -20,18 +20,17 @@ def use_streaming(hook_config, config, log_prefix): # pragma: no cover def dump_data_sources( hook_config, config, - log_prefix, config_paths, borgmatic_runtime_directory, patterns, dry_run, ): ''' - Given a bootstrap configuration dict, a configuration dict, a log prefix, the borgmatic - configuration file paths, the borgmatic runtime directory, the configured patterns, and whether - this is a dry run, create a borgmatic manifest file to store the paths of the configuration - files used to create the archive. But skip this if the bootstrap store_config_files option is - False or if this is a dry run. + Given a bootstrap configuration dict, a configuration dict, the borgmatic configuration file + paths, the borgmatic runtime directory, the configured patterns, and whether this is a dry run, + create a borgmatic manifest file to store the paths of the configuration files used to create + the archive. But skip this if the bootstrap store_config_files option is False or if this is a + dry run. Return an empty sequence, since there are no ongoing dump processes from this hook. ''' @@ -64,11 +63,11 @@ def dump_data_sources( return [] -def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_directory, dry_run): +def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, dry_run): ''' - Given a bootstrap configuration dict, a configuration dict, a log prefix, the borgmatic runtime - directory, and whether this is a dry run, then remove the manifest file created above. If this - is a dry run, then don't actually remove anything. + Given a bootstrap configuration dict, a configuration dict, the borgmatic runtime directory, and + whether this is a dry run, then remove the manifest file created above. If this is a dry run, + then don't actually remove anything. ''' dry_run_label = ' (dry run; not actually removing anything)' if dry_run else '' @@ -79,13 +78,13 @@ def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_ 'bootstrap', ) logger.debug( - f'{log_prefix}: Looking for bootstrap manifest files to remove in {manifest_glob}{dry_run_label}' + f'Looking for bootstrap manifest files to remove in {manifest_glob}{dry_run_label}' ) for manifest_directory in glob.glob(manifest_glob): manifest_file_path = os.path.join(manifest_directory, 'manifest.json') logger.debug( - f'{log_prefix}: Removing bootstrap manifest at {manifest_file_path}{dry_run_label}' + f'Removing bootstrap manifest at {manifest_file_path}{dry_run_label}' ) if dry_run: @@ -103,7 +102,7 @@ def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_ def make_data_source_dump_patterns( - hook_config, config, log_prefix, borgmatic_runtime_directory, name=None + hook_config, config, borgmatic_runtime_directory, name=None ): # pragma: no cover ''' Restores are implemented via the separate, purpose-specific "bootstrap" action rather than the @@ -115,7 +114,6 @@ def make_data_source_dump_patterns( def restore_data_source_dump( hook_config, config, - log_prefix, data_source, dry_run, extract_process, diff --git a/borgmatic/hooks/data_source/btrfs.py b/borgmatic/hooks/data_source/btrfs.py index 5dabedbb..5666a099 100644 --- a/borgmatic/hooks/data_source/btrfs.py +++ b/borgmatic/hooks/data_source/btrfs.py @@ -14,7 +14,7 @@ import borgmatic.hooks.data_source.snapshot logger = logging.getLogger(__name__) -def use_streaming(hook_config, config, log_prefix): # pragma: no cover +def use_streaming(hook_config, config): # pragma: no cover ''' Return whether dump streaming is used for this hook. (Spoiler: It isn't.) ''' @@ -211,26 +211,24 @@ def snapshot_subvolume(btrfs_command, subvolume_path, snapshot_path): # pragma: def dump_data_sources( hook_config, config, - log_prefix, config_paths, borgmatic_runtime_directory, patterns, dry_run, ): ''' - Given a Btrfs configuration dict, a configuration dict, a log prefix, the borgmatic - configuration file paths, the borgmatic runtime directory, the configured patterns, and whether - this is a dry run, auto-detect and snapshot any Btrfs subvolume mount points listed in the given - patterns. Also update those patterns, replacing subvolume mount points with corresponding - snapshot directories so they get stored in the Borg archive instead. Use the log prefix in any - log entries. + Given a Btrfs configuration dict, a configuration dict, the borgmatic configuration file paths, + the borgmatic runtime directory, the configured patterns, and whether this is a dry run, + auto-detect and snapshot any Btrfs subvolume mount points listed in the given patterns. Also + update those patterns, replacing subvolume mount points with corresponding snapshot directories + so they get stored in the Borg archive instead. Return an empty sequence, since there are no ongoing dump processes from this hook. If this is a dry run, then don't actually snapshot anything. ''' dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' - logger.info(f'{log_prefix}: Snapshotting Btrfs subvolumes{dry_run_label}') + logger.info(f'Snapshotting Btrfs subvolumes{dry_run_label}') # Based on the configured patterns, determine Btrfs subvolumes to backup. btrfs_command = hook_config.get('btrfs_command', 'btrfs') @@ -238,11 +236,11 @@ def dump_data_sources( subvolumes = get_subvolumes(btrfs_command, findmnt_command, patterns) if not subvolumes: - logger.warning(f'{log_prefix}: No Btrfs subvolumes found to snapshot{dry_run_label}') + logger.warning(f'No Btrfs subvolumes found to snapshot{dry_run_label}') # Snapshot each subvolume, rewriting patterns to use their snapshot paths. for subvolume in subvolumes: - logger.debug(f'{log_prefix}: Creating Btrfs snapshot for {subvolume.path} subvolume') + logger.debug(f'Creating Btrfs snapshot for {subvolume.path} subvolume') snapshot_path = make_snapshot_path(subvolume.path) @@ -280,12 +278,11 @@ def delete_snapshot(btrfs_command, snapshot_path): # pragma: no cover ) -def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_directory, dry_run): +def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, dry_run): ''' - Given a Btrfs configuration dict, a configuration dict, a log prefix, the borgmatic runtime - directory, and whether this is a dry run, delete any Btrfs snapshots created by borgmatic. Use - the log prefix in any log entries. If this is a dry run or Btrfs isn't configured in borgmatic's - configuration, then don't actually remove anything. + Given a Btrfs configuration dict, a configuration dict, the borgmatic runtime directory, and + whether this is a dry run, delete any Btrfs snapshots created by borgmatic. If this is a dry run + or Btrfs isn't configured in borgmatic's configuration, then don't actually remove anything. ''' if hook_config is None: return @@ -298,10 +295,10 @@ def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_ try: all_subvolumes = get_subvolumes(btrfs_command, findmnt_command) except FileNotFoundError as error: - logger.debug(f'{log_prefix}: Could not find "{error.filename}" command') + logger.debug(f'Could not find "{error.filename}" command') return except subprocess.CalledProcessError as error: - logger.debug(f'{log_prefix}: {error}') + logger.debug(error) return # Reversing the sorted subvolumes ensures that we remove longer mount point paths of child @@ -313,14 +310,14 @@ def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_ ) logger.debug( - f'{log_prefix}: Looking for snapshots to remove in {subvolume_snapshots_glob}{dry_run_label}' + f'Looking for snapshots to remove in {subvolume_snapshots_glob}{dry_run_label}' ) for snapshot_path in glob.glob(subvolume_snapshots_glob): if not os.path.isdir(snapshot_path): continue - logger.debug(f'{log_prefix}: Deleting Btrfs snapshot {snapshot_path}{dry_run_label}') + logger.debug(f'Deleting Btrfs snapshot {snapshot_path}{dry_run_label}') if dry_run: continue @@ -328,10 +325,10 @@ def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_ try: delete_snapshot(btrfs_command, snapshot_path) except FileNotFoundError: - logger.debug(f'{log_prefix}: Could not find "{btrfs_command}" command') + logger.debug(f'Could not find "{btrfs_command}" command') return except subprocess.CalledProcessError as error: - logger.debug(f'{log_prefix}: {error}') + logger.debug(error) return # Strip off the subvolume path from the end of the snapshot path and then delete the @@ -340,7 +337,7 @@ def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_ def make_data_source_dump_patterns( - hook_config, config, log_prefix, borgmatic_runtime_directory, name=None + hook_config, config, borgmatic_runtime_directory, name=None ): # pragma: no cover ''' Restores aren't implemented, because stored files can be extracted directly with "extract". @@ -351,7 +348,6 @@ def make_data_source_dump_patterns( def restore_data_source_dump( hook_config, config, - log_prefix, data_source, dry_run, extract_process, diff --git a/borgmatic/hooks/data_source/dump.py b/borgmatic/hooks/data_source/dump.py index ea3e6a13..11ec2409 100644 --- a/borgmatic/hooks/data_source/dump.py +++ b/borgmatic/hooks/data_source/dump.py @@ -46,14 +46,14 @@ def create_named_pipe_for_dump(dump_path): os.mkfifo(dump_path, mode=0o600) -def remove_data_source_dumps(dump_path, data_source_type_name, log_prefix, dry_run): +def remove_data_source_dumps(dump_path, data_source_type_name, dry_run): ''' Remove all data source dumps in the given dump directory path (including the directory itself). If this is a dry run, then don't actually remove anything. ''' dry_run_label = ' (dry run; not actually removing anything)' if dry_run else '' - logger.debug(f'{log_prefix}: Removing {data_source_type_name} data source dumps{dry_run_label}') + logger.debug(f'Removing {data_source_type_name} data source dumps{dry_run_label}') if dry_run: return diff --git a/borgmatic/hooks/data_source/lvm.py b/borgmatic/hooks/data_source/lvm.py index e804a6ad..958ec8f7 100644 --- a/borgmatic/hooks/data_source/lvm.py +++ b/borgmatic/hooks/data_source/lvm.py @@ -14,7 +14,7 @@ import borgmatic.hooks.data_source.snapshot logger = logging.getLogger(__name__) -def use_streaming(hook_config, config, log_prefix): # pragma: no cover +def use_streaming(hook_config, config): # pragma: no cover ''' Return whether dump streaming is used for this hook. (Spoiler: It isn't.) ''' @@ -161,26 +161,24 @@ DEFAULT_SNAPSHOT_SIZE = '10%ORIGIN' def dump_data_sources( hook_config, config, - log_prefix, config_paths, borgmatic_runtime_directory, patterns, dry_run, ): ''' - Given an LVM configuration dict, a configuration dict, a log prefix, the borgmatic configuration - file paths, the borgmatic runtime directory, the configured patterns, and whether this is a dry - run, auto-detect and snapshot any LVM logical volume mount points listed in the given patterns. - Also update those patterns, replacing logical volume mount points with corresponding snapshot - directories so they get stored in the Borg archive instead. Use the log prefix in any log - entries. + Given an LVM configuration dict, a configuration dict, the borgmatic configuration file paths, + the borgmatic runtime directory, the configured patterns, and whether this is a dry run, + auto-detect and snapshot any LVM logical volume mount points listed in the given patterns. Also + update those patterns, replacing logical volume mount points with corresponding snapshot + directories so they get stored in the Borg archive instead. Return an empty sequence, since there are no ongoing dump processes from this hook. If this is a dry run, then don't actually snapshot anything. ''' dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' - logger.info(f'{log_prefix}: Snapshotting LVM logical volumes{dry_run_label}') + logger.info(f'Snapshotting LVM logical volumes{dry_run_label}') # List logical volumes to get their mount points. lsblk_command = hook_config.get('lsblk_command', 'lsblk') @@ -191,12 +189,12 @@ def dump_data_sources( normalized_runtime_directory = os.path.normpath(borgmatic_runtime_directory) if not requested_logical_volumes: - logger.warning(f'{log_prefix}: No LVM logical volumes found to snapshot{dry_run_label}') + logger.warning(f'No LVM logical volumes found to snapshot{dry_run_label}') for logical_volume in requested_logical_volumes: snapshot_name = f'{logical_volume.name}_{snapshot_suffix}' logger.debug( - f'{log_prefix}: Creating LVM snapshot {snapshot_name} of {logical_volume.mount_point}{dry_run_label}' + f'Creating LVM snapshot {snapshot_name} of {logical_volume.mount_point}{dry_run_label}' ) if not dry_run: @@ -224,7 +222,7 @@ def dump_data_sources( ) logger.debug( - f'{log_prefix}: Mounting LVM snapshot {snapshot_name} at {snapshot_mount_path}{dry_run_label}' + f'Mounting LVM snapshot {snapshot_name} at {snapshot_mount_path}{dry_run_label}' ) if dry_run: @@ -312,12 +310,12 @@ def get_snapshots(lvs_command, snapshot_name=None): raise ValueError(f'Invalid {lvs_command} output: Missing key "{error}"') -def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_directory, dry_run): +def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, dry_run): ''' - Given an LVM configuration dict, a configuration dict, a log prefix, the borgmatic runtime - directory, and whether this is a dry run, unmount and delete any LVM snapshots created by - borgmatic. Use the log prefix in any log entries. If this is a dry run or LVM isn't configured - in borgmatic's configuration, then don't actually remove anything. + Given an LVM configuration dict, a configuration dict, the borgmatic runtime directory, and + whether this is a dry run, unmount and delete any LVM snapshots created by borgmatic. If this is + a dry run or LVM isn't configured in borgmatic's configuration, then don't actually remove + anything. ''' if hook_config is None: return @@ -328,10 +326,10 @@ def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_ try: logical_volumes = get_logical_volumes(hook_config.get('lsblk_command', 'lsblk')) except FileNotFoundError as error: - logger.debug(f'{log_prefix}: Could not find "{error.filename}" command') + logger.debug(f'Could not find "{error.filename}" command') return except subprocess.CalledProcessError as error: - logger.debug(f'{log_prefix}: {error}') + logger.debug(error) return snapshots_glob = os.path.join( @@ -341,7 +339,7 @@ def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_ 'lvm_snapshots', ) logger.debug( - f'{log_prefix}: Looking for snapshots to remove in {snapshots_glob}{dry_run_label}' + f'Looking for snapshots to remove in {snapshots_glob}{dry_run_label}' ) umount_command = hook_config.get('umount_command', 'umount') @@ -367,7 +365,7 @@ def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_ continue logger.debug( - f'{log_prefix}: Unmounting LVM snapshot at {snapshot_mount_path}{dry_run_label}' + f'Unmounting LVM snapshot at {snapshot_mount_path}{dry_run_label}' ) if dry_run: @@ -376,10 +374,10 @@ def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_ try: unmount_snapshot(umount_command, snapshot_mount_path) except FileNotFoundError: - logger.debug(f'{log_prefix}: Could not find "{umount_command}" command') + logger.debug(f'Could not find "{umount_command}" command') return except subprocess.CalledProcessError as error: - logger.debug(f'{log_prefix}: {error}') + logger.debug(error) return if not dry_run: @@ -391,10 +389,10 @@ def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_ try: snapshots = get_snapshots(hook_config.get('lvs_command', 'lvs')) except FileNotFoundError as error: - logger.debug(f'{log_prefix}: Could not find "{error.filename}" command') + logger.debug(f'Could not find "{error.filename}" command') return except subprocess.CalledProcessError as error: - logger.debug(f'{log_prefix}: {error}') + logger.debug(error) return for snapshot in snapshots: @@ -402,14 +400,14 @@ def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_ if not snapshot.name.split('_')[-1].startswith(BORGMATIC_SNAPSHOT_PREFIX): continue - logger.debug(f'{log_prefix}: Deleting LVM snapshot {snapshot.name}{dry_run_label}') + logger.debug(f'Deleting LVM snapshot {snapshot.name}{dry_run_label}') if not dry_run: remove_snapshot(lvremove_command, snapshot.device_path) def make_data_source_dump_patterns( - hook_config, config, log_prefix, borgmatic_runtime_directory, name=None + hook_config, config, borgmatic_runtime_directory, name=None ): # pragma: no cover ''' Restores aren't implemented, because stored files can be extracted directly with "extract". @@ -420,7 +418,6 @@ def make_data_source_dump_patterns( def restore_data_source_dump( hook_config, config, - log_prefix, data_source, dry_run, extract_process, diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index aa84429e..1d2ac8fd 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -25,7 +25,7 @@ def make_dump_path(base_directory): # pragma: no cover SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys') -def database_names_to_dump(database, extra_environment, log_prefix, dry_run): +def database_names_to_dump(database, extra_environment, dry_run): ''' Given a requested database config, return the corresponding sequence of database names to dump. In the case of "all", query for the names of databases on the configured host and return them, @@ -49,7 +49,7 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run): + ('--skip-column-names', '--batch') + ('--execute', 'show schemas') ) - logger.debug(f'{log_prefix}: Querying for "all" MariaDB databases to dump') + logger.debug(f'Querying for "all" MariaDB databases to dump') show_output = execute_command_and_capture_output( show_command, extra_environment=extra_environment ) @@ -62,12 +62,11 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run): def execute_dump_command( - database, log_prefix, dump_path, database_names, extra_environment, dry_run, dry_run_label + database, dump_path, database_names, extra_environment, dry_run, dry_run_label ): ''' Kick off a dump for the given MariaDB database (provided as a configuration dict) to a named - pipe constructed from the given dump path and database name. Use the given log prefix in any - log entries. + pipe constructed from the given dump path and database name. Return a subprocess.Popen instance for the dump process ready to spew to a named pipe. But if this is a dry run, then don't actually dump anything and return None. @@ -82,7 +81,7 @@ def execute_dump_command( if os.path.exists(dump_filename): logger.warning( - f'{log_prefix}: Skipping duplicate dump of MariaDB database "{database_name}" to {dump_filename}' + f'Skipping duplicate dump of MariaDB database "{database_name}" to {dump_filename}' ) return None @@ -104,7 +103,7 @@ def execute_dump_command( ) logger.debug( - f'{log_prefix}: Dumping MariaDB database "{database_name}" to {dump_filename}{dry_run_label}' + f'Dumping MariaDB database "{database_name}" to {dump_filename}{dry_run_label}' ) if dry_run: return None @@ -118,14 +117,14 @@ def execute_dump_command( ) -def get_default_port(databases, config, log_prefix): # pragma: no cover +def get_default_port(databases, config): # pragma: no cover return 3306 -def use_streaming(databases, config, log_prefix): +def use_streaming(databases, config): ''' - Given a sequence of MariaDB database configuration dicts, a configuration dict (ignored), and a - log prefix (ignored), return whether streaming will be using during dumps. + Given a sequence of MariaDB database configuration dicts, a configuration dict (ignored), return + whether streaming will be using during dumps. ''' return any(databases) @@ -133,7 +132,6 @@ def use_streaming(databases, config, log_prefix): def dump_data_sources( databases, config, - log_prefix, config_paths, borgmatic_runtime_directory, patterns, @@ -142,8 +140,7 @@ def dump_data_sources( ''' Dump the given MariaDB databases to a named pipe. The databases are supplied as a sequence of dicts, one dict describing each database as per the configuration schema. Use the given - borgmatic runtime directory to construct the destination path and the given log prefix in any - log entries. + borgmatic runtime directory to construct the destination path. Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence. @@ -153,13 +150,13 @@ def dump_data_sources( dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' processes = [] - logger.info(f'{log_prefix}: Dumping MariaDB databases{dry_run_label}') + logger.info(f'Dumping MariaDB databases{dry_run_label}') for database in databases: dump_path = make_dump_path(borgmatic_runtime_directory) extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None dump_database_names = database_names_to_dump( - database, extra_environment, log_prefix, dry_run + database, extra_environment, dry_run ) if not dump_database_names: @@ -175,7 +172,6 @@ def dump_data_sources( processes.append( execute_dump_command( renamed_database, - log_prefix, dump_path, (dump_name,), extra_environment, @@ -187,7 +183,6 @@ def dump_data_sources( processes.append( execute_dump_command( database, - log_prefix, dump_path, dump_database_names, extra_environment, @@ -207,25 +202,25 @@ def dump_data_sources( def remove_data_source_dumps( - databases, config, log_prefix, borgmatic_runtime_directory, dry_run + databases, config, borgmatic_runtime_directory, dry_run ): # pragma: no cover ''' Remove all database dump files for this hook regardless of the given databases. Use the - borgmatic_runtime_directory to construct the destination path and the log prefix in any log - entries. If this is a dry run, then don't actually remove anything. + borgmatic_runtime_directory to construct the destination path. If this is a dry run, then don't + actually remove anything. ''' dump.remove_data_source_dumps( - make_dump_path(borgmatic_runtime_directory), 'MariaDB', log_prefix, dry_run + make_dump_path(borgmatic_runtime_directory), 'MariaDB', dry_run ) def make_data_source_dump_patterns( - databases, config, log_prefix, borgmatic_runtime_directory, name=None + databases, config, borgmatic_runtime_directory, name=None ): # pragma: no cover ''' - Given a sequence of configurations dicts, a configuration dict, a prefix to log with, the - borgmatic runtime directory, and a database name to match, return the corresponding glob - patterns to match the database dump in an archive. + Given a sequence of configurations dicts, a configuration dict, the borgmatic runtime directory, + and a database name to match, return the corresponding glob patterns to match the database dump + in an archive. ''' borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config) @@ -243,7 +238,6 @@ def make_data_source_dump_patterns( def restore_data_source_dump( hook_config, config, - log_prefix, data_source, dry_run, extract_process, @@ -252,9 +246,9 @@ def restore_data_source_dump( ): ''' Restore a database from the given extract stream. The database is supplied as a data source - configuration dict, but the given hook configuration is ignored. The given log prefix is used - for any log entries. If this is a dry run, then don't actually restore anything. Trigger the - given active extract process (an instance of subprocess.Popen) to produce output to consume. + configuration dict, but the given hook configuration is ignored. If this is a dry run, then + don't actually restore anything. Trigger the given active extract process (an instance of + subprocess.Popen) to produce output to consume. ''' dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' hostname = connection_params['hostname'] or data_source.get( @@ -288,7 +282,7 @@ def restore_data_source_dump( ) extra_environment = {'MYSQL_PWD': password} if password else None - logger.debug(f"{log_prefix}: Restoring MariaDB database {data_source['name']}{dry_run_label}") + logger.debug(f"Restoring MariaDB database {data_source['name']}{dry_run_label}") if dry_run: return diff --git a/borgmatic/hooks/data_source/mongodb.py b/borgmatic/hooks/data_source/mongodb.py index 18ce6561..b5c2c744 100644 --- a/borgmatic/hooks/data_source/mongodb.py +++ b/borgmatic/hooks/data_source/mongodb.py @@ -17,14 +17,14 @@ def make_dump_path(base_directory): # pragma: no cover return dump.make_data_source_dump_path(base_directory, 'mongodb_databases') -def get_default_port(databases, config, log_prefix): # pragma: no cover +def get_default_port(databases, config): # pragma: no cover return 27017 -def use_streaming(databases, config, log_prefix): +def use_streaming(databases, config): ''' - Given a sequence of MongoDB database configuration dicts, a configuration dict (ignored), and a - log prefix (ignored), return whether streaming will be using during dumps. + Given a sequence of MongoDB database configuration dicts, a configuration dict (ignored), return + whether streaming will be using during dumps. ''' return any(database.get('format') != 'directory' for database in databases) @@ -32,7 +32,6 @@ def use_streaming(databases, config, log_prefix): def dump_data_sources( databases, config, - log_prefix, config_paths, borgmatic_runtime_directory, patterns, @@ -41,8 +40,7 @@ def dump_data_sources( ''' Dump the given MongoDB databases to a named pipe. The databases are supplied as a sequence of dicts, one dict describing each database as per the configuration schema. Use the borgmatic - runtime directory to construct the destination path (used for the directory format and the given - log prefix in any log entries. + runtime directory to construct the destination path (used for the directory format. Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence. @@ -51,7 +49,7 @@ def dump_data_sources( ''' dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' - logger.info(f'{log_prefix}: Dumping MongoDB databases{dry_run_label}') + logger.info(f'Dumping MongoDB databases{dry_run_label}') processes = [] for database in databases: @@ -65,7 +63,7 @@ def dump_data_sources( dump_format = database.get('format', 'archive') logger.debug( - f'{log_prefix}: Dumping MongoDB database {name} to {dump_filename}{dry_run_label}', + f'Dumping MongoDB database {name} to {dump_filename}{dry_run_label}', ) if dry_run: continue @@ -118,25 +116,25 @@ def build_dump_command(database, dump_filename, dump_format): def remove_data_source_dumps( - databases, config, log_prefix, borgmatic_runtime_directory, dry_run + databases, config, borgmatic_runtime_directory, dry_run ): # pragma: no cover ''' Remove all database dump files for this hook regardless of the given databases. Use the - borgmatic_runtime_directory to construct the destination path and the log prefix in any log - entries. If this is a dry run, then don't actually remove anything. + borgmatic_runtime_directory to construct the destination path. If this is a dry run, then don't + actually remove anything. ''' dump.remove_data_source_dumps( - make_dump_path(borgmatic_runtime_directory), 'MongoDB', log_prefix, dry_run + make_dump_path(borgmatic_runtime_directory), 'MongoDB', dry_run ) def make_data_source_dump_patterns( - databases, config, log_prefix, borgmatic_runtime_directory, name=None + databases, config, borgmatic_runtime_directory, name=None ): # pragma: no cover ''' - Given a sequence of configurations dicts, a configuration dict, a prefix to log with, the - borgmatic runtime directory, and a database name to match, return the corresponding glob - patterns to match the database dump in an archive. + Given a sequence of configurations dicts, a configuration dict, the borgmatic runtime directory, + and a database name to match, return the corresponding glob patterns to match the database dump + in an archive. ''' borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config) @@ -154,7 +152,6 @@ def make_data_source_dump_patterns( def restore_data_source_dump( hook_config, config, - log_prefix, data_source, dry_run, extract_process, @@ -164,9 +161,9 @@ def restore_data_source_dump( ''' Restore a database from the given extract stream. The database is supplied as a data source configuration dict, but the given hook configuration is ignored. The given configuration dict is - used to construct the destination path, and the given log prefix is used for any log entries. If - this is a dry run, then don't actually restore anything. Trigger the given active extract - process (an instance of subprocess.Popen) to produce output to consume. + used to construct the destination path. If this is a dry run, then don't actually restore + anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce + output to consume. If the extract process is None, then restore the dump from the filesystem rather than from an extract stream. @@ -181,7 +178,7 @@ def restore_data_source_dump( extract_process, data_source, dump_filename, connection_params ) - logger.debug(f"{log_prefix}: Restoring MongoDB database {data_source['name']}{dry_run_label}") + logger.debug(f"Restoring MongoDB database {data_source['name']}{dry_run_label}") if dry_run: return diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index febd2afc..40d9d959 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -25,7 +25,7 @@ def make_dump_path(base_directory): # pragma: no cover SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys') -def database_names_to_dump(database, extra_environment, log_prefix, dry_run): +def database_names_to_dump(database, extra_environment, dry_run): ''' Given a requested database config, return the corresponding sequence of database names to dump. In the case of "all", query for the names of databases on the configured host and return them, @@ -49,7 +49,7 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run): + ('--skip-column-names', '--batch') + ('--execute', 'show schemas') ) - logger.debug(f'{log_prefix}: Querying for "all" MySQL databases to dump') + logger.debug(f'Querying for "all" MySQL databases to dump') show_output = execute_command_and_capture_output( show_command, extra_environment=extra_environment ) @@ -62,12 +62,11 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run): def execute_dump_command( - database, log_prefix, dump_path, database_names, extra_environment, dry_run, dry_run_label + database, dump_path, database_names, extra_environment, dry_run, dry_run_label ): ''' Kick off a dump for the given MySQL/MariaDB database (provided as a configuration dict) to a - named pipe constructed from the given dump path and database name. Use the given log prefix in - any log entries. + named pipe constructed from the given dump path and database name. Return a subprocess.Popen instance for the dump process ready to spew to a named pipe. But if this is a dry run, then don't actually dump anything and return None. @@ -82,7 +81,7 @@ def execute_dump_command( if os.path.exists(dump_filename): logger.warning( - f'{log_prefix}: Skipping duplicate dump of MySQL database "{database_name}" to {dump_filename}' + f'Skipping duplicate dump of MySQL database "{database_name}" to {dump_filename}' ) return None @@ -103,7 +102,7 @@ def execute_dump_command( ) logger.debug( - f'{log_prefix}: Dumping MySQL database "{database_name}" to {dump_filename}{dry_run_label}' + f'Dumping MySQL database "{database_name}" to {dump_filename}{dry_run_label}' ) if dry_run: return None @@ -117,14 +116,14 @@ def execute_dump_command( ) -def get_default_port(databases, config, log_prefix): # pragma: no cover +def get_default_port(databases, config): # pragma: no cover return 3306 -def use_streaming(databases, config, log_prefix): +def use_streaming(databases, config): ''' - Given a sequence of MySQL database configuration dicts, a configuration dict (ignored), and a - log prefix (ignored), return whether streaming will be using during dumps. + Given a sequence of MySQL database configuration dicts, a configuration dict (ignored), return + whether streaming will be using during dumps. ''' return any(databases) @@ -132,7 +131,6 @@ def use_streaming(databases, config, log_prefix): def dump_data_sources( databases, config, - log_prefix, config_paths, borgmatic_runtime_directory, patterns, @@ -141,8 +139,7 @@ def dump_data_sources( ''' Dump the given MySQL/MariaDB databases to a named pipe. The databases are supplied as a sequence of dicts, one dict describing each database as per the configuration schema. Use the given - borgmatic runtime directory to construct the destination path and the given log prefix in any - log entries. + borgmatic runtime directory to construct the destination path. Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence. @@ -152,13 +149,13 @@ def dump_data_sources( dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' processes = [] - logger.info(f'{log_prefix}: Dumping MySQL databases{dry_run_label}') + logger.info(f'Dumping MySQL databases{dry_run_label}') for database in databases: dump_path = make_dump_path(borgmatic_runtime_directory) extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None dump_database_names = database_names_to_dump( - database, extra_environment, log_prefix, dry_run + database, extra_environment, dry_run ) if not dump_database_names: @@ -174,7 +171,6 @@ def dump_data_sources( processes.append( execute_dump_command( renamed_database, - log_prefix, dump_path, (dump_name,), extra_environment, @@ -186,7 +182,6 @@ def dump_data_sources( processes.append( execute_dump_command( database, - log_prefix, dump_path, dump_database_names, extra_environment, @@ -206,25 +201,25 @@ def dump_data_sources( def remove_data_source_dumps( - databases, config, log_prefix, borgmatic_runtime_directory, dry_run + databases, config, borgmatic_runtime_directory, dry_run ): # pragma: no cover ''' Remove all database dump files for this hook regardless of the given databases. Use the - borgmatic runtime directory to construct the destination path and the log prefix in any log - entries. If this is a dry run, then don't actually remove anything. + borgmatic runtime directory to construct the destination path. If this is a dry run, then don't + actually remove anything. ''' dump.remove_data_source_dumps( - make_dump_path(borgmatic_runtime_directory), 'MySQL', log_prefix, dry_run + make_dump_path(borgmatic_runtime_directory), 'MySQL', dry_run ) def make_data_source_dump_patterns( - databases, config, log_prefix, borgmatic_runtime_directory, name=None + databases, config, borgmatic_runtime_directory, name=None ): # pragma: no cover ''' - Given a sequence of configurations dicts, a configuration dict, a prefix to log with, the - borgmatic runtime directory, and a database name to match, return the corresponding glob - patterns to match the database dump in an archive. + Given a sequence of configurations dicts, a configuration dict, the borgmatic runtime directory, + and a database name to match, return the corresponding glob patterns to match the database dump + in an archive. ''' borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config) @@ -242,7 +237,6 @@ def make_data_source_dump_patterns( def restore_data_source_dump( hook_config, config, - log_prefix, data_source, dry_run, extract_process, @@ -251,9 +245,9 @@ def restore_data_source_dump( ): ''' Restore a database from the given extract stream. The database is supplied as a data source - configuration dict, but the given hook configuration is ignored. The given log prefix is used - for any log entries. If this is a dry run, then don't actually restore anything. Trigger the - given active extract process (an instance of subprocess.Popen) to produce output to consume. + configuration dict, but the given hook configuration is ignored. If this is a dry run, then + don't actually restore anything. Trigger the given active extract process (an instance of + subprocess.Popen) to produce output to consume. ''' dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' hostname = connection_params['hostname'] or data_source.get( @@ -287,7 +281,7 @@ def restore_data_source_dump( ) extra_environment = {'MYSQL_PWD': password} if password else None - logger.debug(f"{log_prefix}: Restoring MySQL database {data_source['name']}{dry_run_label}") + logger.debug(f"Restoring MySQL database {data_source['name']}{dry_run_label}") if dry_run: return diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index bb5e5a45..95022264 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -58,7 +58,7 @@ def make_extra_environment(database, restore_connection_params=None): EXCLUDED_DATABASE_NAMES = ('template0', 'template1') -def database_names_to_dump(database, extra_environment, log_prefix, dry_run): +def database_names_to_dump(database, extra_environment, dry_run): ''' Given a requested database config, return the corresponding sequence of database names to dump. In the case of "all" when a database format is given, query for the names of databases on the @@ -85,7 +85,7 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run): + (('--username', database['username']) if 'username' in database else ()) + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ()) ) - logger.debug(f'{log_prefix}: Querying for "all" PostgreSQL databases to dump') + logger.debug(f'Querying for "all" PostgreSQL databases to dump') list_output = execute_command_and_capture_output( list_command, extra_environment=extra_environment ) @@ -97,14 +97,14 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run): ) -def get_default_port(databases, config, log_prefix): # pragma: no cover +def get_default_port(databases, config): # pragma: no cover return 5432 -def use_streaming(databases, config, log_prefix): +def use_streaming(databases, config): ''' - Given a sequence of PostgreSQL database configuration dicts, a configuration dict (ignored), and - a log prefix (ignored), return whether streaming will be using during dumps. + Given a sequence of PostgreSQL database configuration dicts, a configuration dict (ignored), + return whether streaming will be using during dumps. ''' return any(database.get('format') != 'directory' for database in databases) @@ -112,7 +112,6 @@ def use_streaming(databases, config, log_prefix): def dump_data_sources( databases, config, - log_prefix, config_paths, borgmatic_runtime_directory, patterns, @@ -121,8 +120,7 @@ def dump_data_sources( ''' Dump the given PostgreSQL databases to a named pipe. The databases are supplied as a sequence of dicts, one dict describing each database as per the configuration schema. Use the given - borgmatic runtime directory to construct the destination path and the given log prefix in any - log entries. + borgmatic runtime directory to construct the destination path. Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence. @@ -134,13 +132,13 @@ def dump_data_sources( dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' processes = [] - logger.info(f'{log_prefix}: Dumping PostgreSQL databases{dry_run_label}') + logger.info(f'Dumping PostgreSQL databases{dry_run_label}') for database in databases: extra_environment = make_extra_environment(database) dump_path = make_dump_path(borgmatic_runtime_directory) dump_database_names = database_names_to_dump( - database, extra_environment, log_prefix, dry_run + database, extra_environment, dry_run ) if not dump_database_names: @@ -164,7 +162,7 @@ def dump_data_sources( ) if os.path.exists(dump_filename): logger.warning( - f'{log_prefix}: Skipping duplicate dump of PostgreSQL database "{database_name}" to {dump_filename}' + f'Skipping duplicate dump of PostgreSQL database "{database_name}" to {dump_filename}' ) continue @@ -198,7 +196,7 @@ def dump_data_sources( ) logger.debug( - f'{log_prefix}: Dumping PostgreSQL database "{database_name}" to {dump_filename}{dry_run_label}' + f'Dumping PostgreSQL database "{database_name}" to {dump_filename}{dry_run_label}' ) if dry_run: continue @@ -232,25 +230,25 @@ def dump_data_sources( def remove_data_source_dumps( - databases, config, log_prefix, borgmatic_runtime_directory, dry_run + databases, config, borgmatic_runtime_directory, dry_run ): # pragma: no cover ''' Remove all database dump files for this hook regardless of the given databases. Use the - borgmatic runtime directory to construct the destination path and the log prefix in any log - entries. If this is a dry run, then don't actually remove anything. + borgmatic runtime directory to construct the destination path. If this is a dry run, then don't + actually remove anything. ''' dump.remove_data_source_dumps( - make_dump_path(borgmatic_runtime_directory), 'PostgreSQL', log_prefix, dry_run + make_dump_path(borgmatic_runtime_directory), 'PostgreSQL', dry_run ) def make_data_source_dump_patterns( - databases, config, log_prefix, borgmatic_runtime_directory, name=None + databases, config, borgmatic_runtime_directory, name=None ): # pragma: no cover ''' - Given a sequence of configurations dicts, a configuration dict, a prefix to log with, the - borgmatic runtime directory, and a database name to match, return the corresponding glob - patterns to match the database dump in an archive. + Given a sequence of configurations dicts, a configuration dict, the borgmatic runtime directory, + and a database name to match, return the corresponding glob patterns to match the database dump + in an archive. ''' borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config) @@ -268,7 +266,6 @@ def make_data_source_dump_patterns( def restore_data_source_dump( hook_config, config, - log_prefix, data_source, dry_run, extract_process, @@ -278,10 +275,9 @@ def restore_data_source_dump( ''' Restore a database from the given extract stream. The database is supplied as a data source configuration dict, but the given hook configuration is ignored. The given borgmatic runtime - directory is used to construct the destination path (used for the directory format), and the - given log prefix is used for any log entries. If this is a dry run, then don't actually restore - anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce - output to consume. + directory is used to construct the destination path (used for the directory format). If this is + a dry run, then don't actually restore anything. Trigger the given active extract process (an + instance of subprocess.Popen) to produce output to consume. If the extract process is None, then restore the dump from the filesystem rather than from an extract stream. @@ -355,7 +351,7 @@ def restore_data_source_dump( ) logger.debug( - f"{log_prefix}: Restoring PostgreSQL database {data_source['name']}{dry_run_label}" + f"Restoring PostgreSQL database {data_source['name']}{dry_run_label}" ) if dry_run: return diff --git a/borgmatic/hooks/data_source/sqlite.py b/borgmatic/hooks/data_source/sqlite.py index d901f3f7..15b02bc9 100644 --- a/borgmatic/hooks/data_source/sqlite.py +++ b/borgmatic/hooks/data_source/sqlite.py @@ -17,14 +17,14 @@ def make_dump_path(base_directory): # pragma: no cover return dump.make_data_source_dump_path(base_directory, 'sqlite_databases') -def get_default_port(databases, config, log_prefix): # pragma: no cover +def get_default_port(databases, config): # pragma: no cover return None # SQLite doesn't use a port. -def use_streaming(databases, config, log_prefix): +def use_streaming(databases, config): ''' - Given a sequence of SQLite database configuration dicts, a configuration dict (ignored), and a - log prefix (ignored), return whether streaming will be using during dumps. + Given a sequence of SQLite database configuration dicts, a configuration dict (ignored), return + whether streaming will be using during dumps. ''' return any(databases) @@ -32,7 +32,6 @@ def use_streaming(databases, config, log_prefix): def dump_data_sources( databases, config, - log_prefix, config_paths, borgmatic_runtime_directory, patterns, @@ -41,7 +40,7 @@ def dump_data_sources( ''' Dump the given SQLite databases to a named pipe. The databases are supplied as a sequence of configuration dicts, as per the configuration schema. Use the given borgmatic runtime directory - to construct the destination path and the given log prefix in any log entries. + to construct the destination path. Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence. @@ -51,7 +50,7 @@ def dump_data_sources( dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else '' processes = [] - logger.info(f'{log_prefix}: Dumping SQLite databases{dry_run_label}') + logger.info(f'Dumping SQLite databases{dry_run_label}') for database in databases: database_path = database['path'] @@ -60,7 +59,7 @@ def dump_data_sources( logger.warning('The "all" database name has no meaning for SQLite databases') if not os.path.exists(database_path): logger.warning( - f'{log_prefix}: No SQLite database at {database_path}; an empty database will be created and dumped' + f'No SQLite database at {database_path}; an empty database will be created and dumped' ) dump_path = make_dump_path(borgmatic_runtime_directory) @@ -68,7 +67,7 @@ def dump_data_sources( if os.path.exists(dump_filename): logger.warning( - f'{log_prefix}: Skipping duplicate dump of SQLite database at {database_path} to {dump_filename}' + f'Skipping duplicate dump of SQLite database at {database_path} to {dump_filename}' ) continue @@ -80,7 +79,7 @@ def dump_data_sources( shlex.quote(dump_filename), ) logger.debug( - f'{log_prefix}: Dumping SQLite database at {database_path} to {dump_filename}{dry_run_label}' + f'Dumping SQLite database at {database_path} to {dump_filename}{dry_run_label}' ) if dry_run: continue @@ -99,25 +98,25 @@ def dump_data_sources( def remove_data_source_dumps( - databases, config, log_prefix, borgmatic_runtime_directory, dry_run + databases, config, borgmatic_runtime_directory, dry_run ): # pragma: no cover ''' Remove all database dump files for this hook regardless of the given databases. Use the - borgmatic runtime directory to construct the destination path and the log prefix in any log - entries. If this is a dry run, then don't actually remove anything. + borgmatic runtime directory to construct the destination path. If this is a dry run, then don't + actually remove anything. ''' dump.remove_data_source_dumps( - make_dump_path(borgmatic_runtime_directory), 'SQLite', log_prefix, dry_run + make_dump_path(borgmatic_runtime_directory), 'SQLite', dry_run ) def make_data_source_dump_patterns( - databases, config, log_prefix, borgmatic_runtime_directory, name=None + databases, config, borgmatic_runtime_directory, name=None ): # pragma: no cover ''' - Given a sequence of configurations dicts, a configuration dict, a prefix to log with, the - borgmatic runtime directory, and a database name to match, return the corresponding glob - patterns to match the database dump in an archive. + Given a sequence of configurations dicts, a configuration dict, the borgmatic runtime directory, + and a database name to match, return the corresponding glob patterns to match the database dump + in an archive. ''' borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config) @@ -135,7 +134,6 @@ def make_data_source_dump_patterns( def restore_data_source_dump( hook_config, config, - log_prefix, data_source, dry_run, extract_process, @@ -144,22 +142,22 @@ def restore_data_source_dump( ): ''' Restore a database from the given extract stream. The database is supplied as a data source - configuration dict, but the given hook configuration is ignored. The given log prefix is used - for any log entries. If this is a dry run, then don't actually restore anything. Trigger the - given active extract process (an instance of subprocess.Popen) to produce output to consume. + configuration dict, but the given hook configuration is ignored. If this is a dry run, then + don't actually restore anything. Trigger the given active extract process (an instance of + subprocess.Popen) to produce output to consume. ''' dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else '' database_path = connection_params['restore_path'] or data_source.get( 'restore_path', data_source.get('path') ) - logger.debug(f'{log_prefix}: Restoring SQLite database at {database_path}{dry_run_label}') + logger.debug(f'Restoring SQLite database at {database_path}{dry_run_label}') if dry_run: return try: os.remove(database_path) - logger.warning(f'{log_prefix}: Removed existing SQLite database at {database_path}') + logger.warning(f'Removed existing SQLite database at {database_path}') except FileNotFoundError: # pragma: no cover pass diff --git a/borgmatic/hooks/data_source/zfs.py b/borgmatic/hooks/data_source/zfs.py index 184fa78e..c1ddb38c 100644 --- a/borgmatic/hooks/data_source/zfs.py +++ b/borgmatic/hooks/data_source/zfs.py @@ -13,7 +13,7 @@ import borgmatic.hooks.data_source.snapshot logger = logging.getLogger(__name__) -def use_streaming(hook_config, config, log_prefix): # pragma: no cover +def use_streaming(hook_config, config): # pragma: no cover ''' Return whether dump streaming is used for this hook. (Spoiler: It isn't.) ''' @@ -189,26 +189,25 @@ def make_borg_snapshot_pattern(pattern, normalized_runtime_directory): def dump_data_sources( hook_config, config, - log_prefix, config_paths, borgmatic_runtime_directory, patterns, dry_run, ): ''' - Given a ZFS configuration dict, a configuration dict, a log prefix, the borgmatic configuration - file paths, the borgmatic runtime directory, the configured patterns, and whether this is a dry - run, auto-detect and snapshot any ZFS dataset mount points listed in the given patterns and any + Given a ZFS configuration dict, a configuration dict, the borgmatic configuration file paths, + the borgmatic runtime directory, the configured patterns, and whether this is a dry run, + auto-detect and snapshot any ZFS dataset mount points listed in the given patterns and any dataset with a borgmatic-specific user property. Also update those patterns, replacing dataset mount points with corresponding snapshot directories so they get stored in the Borg archive - instead. Use the log prefix in any log entries. + instead. Return an empty sequence, since there are no ongoing dump processes from this hook. If this is a dry run, then don't actually snapshot anything. ''' dry_run_label = ' (dry run; not actually snapshotting anything)' if dry_run else '' - logger.info(f'{log_prefix}: Snapshotting ZFS datasets{dry_run_label}') + logger.info(f'Snapshotting ZFS datasets{dry_run_label}') # List ZFS datasets to get their mount points. zfs_command = hook_config.get('zfs_command', 'zfs') @@ -219,12 +218,12 @@ def dump_data_sources( normalized_runtime_directory = os.path.normpath(borgmatic_runtime_directory) if not requested_datasets: - logger.warning(f'{log_prefix}: No ZFS datasets found to snapshot{dry_run_label}') + logger.warning(f'No ZFS datasets found to snapshot{dry_run_label}') for dataset in requested_datasets: full_snapshot_name = f'{dataset.name}@{snapshot_name}' logger.debug( - f'{log_prefix}: Creating ZFS snapshot {full_snapshot_name} of {dataset.mount_point}{dry_run_label}' + f'Creating ZFS snapshot {full_snapshot_name} of {dataset.mount_point}{dry_run_label}' ) if not dry_run: @@ -239,7 +238,7 @@ def dump_data_sources( ) logger.debug( - f'{log_prefix}: Mounting ZFS snapshot {full_snapshot_name} at {snapshot_mount_path}{dry_run_label}' + f'Mounting ZFS snapshot {full_snapshot_name} at {snapshot_mount_path}{dry_run_label}' ) if dry_run: @@ -306,12 +305,12 @@ def get_all_snapshots(zfs_command): return tuple(line.rstrip() for line in list_output.splitlines()) -def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_directory, dry_run): +def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, dry_run): ''' - Given a ZFS configuration dict, a configuration dict, a log prefix, the borgmatic runtime - directory, and whether this is a dry run, unmount and destroy any ZFS snapshots created by - borgmatic. Use the log prefix in any log entries. If this is a dry run or ZFS isn't configured - in borgmatic's configuration, then don't actually remove anything. + Given a ZFS configuration dict, a configuration dict, the borgmatic runtime directory, and + whether this is a dry run, unmount and destroy any ZFS snapshots created by borgmatic. If this + is a dry run or ZFS isn't configured in borgmatic's configuration, then don't actually remove + anything. ''' if hook_config is None: return @@ -324,10 +323,10 @@ def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_ try: dataset_mount_points = get_all_dataset_mount_points(zfs_command) except FileNotFoundError: - logger.debug(f'{log_prefix}: Could not find "{zfs_command}" command') + logger.debug(f'Could not find "{zfs_command}" command') return except subprocess.CalledProcessError as error: - logger.debug(f'{log_prefix}: {error}') + logger.debug(error) return snapshots_glob = os.path.join( @@ -337,7 +336,7 @@ def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_ 'zfs_snapshots', ) logger.debug( - f'{log_prefix}: Looking for snapshots to remove in {snapshots_glob}{dry_run_label}' + f'Looking for snapshots to remove in {snapshots_glob}{dry_run_label}' ) umount_command = hook_config.get('umount_command', 'umount') @@ -364,17 +363,17 @@ def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_ continue logger.debug( - f'{log_prefix}: Unmounting ZFS snapshot at {snapshot_mount_path}{dry_run_label}' + f'Unmounting ZFS snapshot at {snapshot_mount_path}{dry_run_label}' ) if not dry_run: try: unmount_snapshot(umount_command, snapshot_mount_path) except FileNotFoundError: - logger.debug(f'{log_prefix}: Could not find "{umount_command}" command') + logger.debug(f'Could not find "{umount_command}" command') return except subprocess.CalledProcessError as error: - logger.debug(f'{log_prefix}: {error}') + logger.debug(error) return if not dry_run: @@ -388,14 +387,14 @@ def remove_data_source_dumps(hook_config, config, log_prefix, borgmatic_runtime_ if not full_snapshot_name.split('@')[-1].startswith(BORGMATIC_SNAPSHOT_PREFIX): continue - logger.debug(f'{log_prefix}: Destroying ZFS snapshot {full_snapshot_name}{dry_run_label}') + logger.debug(f'Destroying ZFS snapshot {full_snapshot_name}{dry_run_label}') if not dry_run: destroy_snapshot(zfs_command, full_snapshot_name) def make_data_source_dump_patterns( - hook_config, config, log_prefix, borgmatic_runtime_directory, name=None + hook_config, config, borgmatic_runtime_directory, name=None ): # pragma: no cover ''' Restores aren't implemented, because stored files can be extracted directly with "extract". @@ -406,7 +405,6 @@ def make_data_source_dump_patterns( def restore_data_source_dump( hook_config, config, - log_prefix, data_source, dry_run, extract_process, diff --git a/borgmatic/hooks/dispatch.py b/borgmatic/hooks/dispatch.py index d3c828c4..d806fa68 100644 --- a/borgmatic/hooks/dispatch.py +++ b/borgmatic/hooks/dispatch.py @@ -21,12 +21,12 @@ def get_submodule_names(parent_module): # pragma: no cover return tuple(module_info.name for module_info in pkgutil.iter_modules(parent_module.__path__)) -def call_hook(function_name, config, log_prefix, hook_name, *args, **kwargs): +def call_hook(function_name, config, hook_name, *args, **kwargs): ''' - Given a configuration dict and a prefix to use in log entries, call the requested function of - the Python module corresponding to the given hook name. Supply that call with the configuration - for this hook (if any), the log prefix, and any given args and kwargs. Return the return value - of that call or None if the module in question is not a hook. + Given a configuration dict, call the requested function of the Python module corresponding to + the given hook name. Supply that call with the configuration for this hook (if any) and any + given args and kwargs. Return the return value of that call or None if the module in question is + not a hook. Raise ValueError if the hook name is unknown. Raise AttributeError if the function name is not found in the module. @@ -54,17 +54,16 @@ def call_hook(function_name, config, log_prefix, hook_name, *args, **kwargs): else: raise ValueError(f'Unknown hook name: {hook_name}') - logger.debug(f'{log_prefix}: Calling {hook_name} hook function {function_name}') + logger.debug(f'Calling {hook_name} hook function {function_name}') - return getattr(module, function_name)(hook_config, config, log_prefix, *args, **kwargs) + return getattr(module, function_name)(hook_config, config, *args, **kwargs) -def call_hooks(function_name, config, log_prefix, hook_type, *args, **kwargs): +def call_hooks(function_name, config, hook_type, *args, **kwargs): ''' - Given a configuration dict and a prefix to use in log entries, call the requested function of - the Python module corresponding to each hook of the given hook type (either "data_source" or - "monitoring"). Supply each call with the configuration for that hook, the log prefix, and any - given args and kwargs. + Given a configuration dict, call the requested function of the Python module corresponding to + each hook of the given hook type (either "data_source" or "monitoring"). Supply each call with + the configuration for that hook, and any given args and kwargs. Collect any return values into a dict from module name to return value. Note that the module name is the name of the hook module itself, which might be different from the hook configuration @@ -78,7 +77,7 @@ def call_hooks(function_name, config, log_prefix, hook_type, *args, **kwargs): Raise anything else that a called function raises. An error stops calls to subsequent functions. ''' return { - hook_name: call_hook(function_name, config, log_prefix, hook_name, *args, **kwargs) + hook_name: call_hook(function_name, config, hook_name, *args, **kwargs) for hook_name in get_submodule_names( importlib.import_module(f'borgmatic.hooks.{hook_type.value}') ) @@ -86,18 +85,18 @@ def call_hooks(function_name, config, log_prefix, hook_type, *args, **kwargs): } -def call_hooks_even_if_unconfigured(function_name, config, log_prefix, hook_type, *args, **kwargs): +def call_hooks_even_if_unconfigured(function_name, config, hook_type, *args, **kwargs): ''' - Given a configuration dict and a prefix to use in log entries, call the requested function of - the Python module corresponding to each hook of the given hook type (either "data_source" or - "monitoring"). Supply each call with the configuration for that hook, the log prefix, and any - given args and kwargs. Collect any return values into a dict from hook name to return value. + Given a configuration dict, call the requested function of the Python module corresponding to + each hook of the given hook type (either "data_source" or "monitoring"). Supply each call with + the configuration for that hook and any given args and kwargs. Collect any return values into a + dict from hook name to return value. Raise AttributeError if the function name is not found in the module. Raise anything else that a called function raises. An error stops calls to subsequent functions. ''' return { - hook_name: call_hook(function_name, config, log_prefix, hook_name, *args, **kwargs) + hook_name: call_hook(function_name, config, hook_name, *args, **kwargs) for hook_name in get_submodule_names( importlib.import_module(f'borgmatic.hooks.{hook_type.value}') ) diff --git a/borgmatic/logger.py b/borgmatic/logger.py index 1114e7f1..888206a2 100644 --- a/borgmatic/logger.py +++ b/borgmatic/logger.py @@ -88,8 +88,8 @@ class Multi_stream_handler(logging.Handler): class Console_no_color_formatter(logging.Formatter): - def format(self, record): # pragma: no cover - return record.msg + def __init__(self, *args, **kwargs): + super(Console_no_color_formatter, self).__init__('{prefix}{message}', style='{', defaults={'prefix': ''}, *args, **kwargs) class Color(enum.Enum): @@ -102,6 +102,9 @@ class Color(enum.Enum): class Console_color_formatter(logging.Formatter): + def __init__(self, *args, **kwargs): + super(Console_color_formatter, self).__init__('{prefix}{message}', style='{', defaults={'prefix': ''}, *args, **kwargs) + def format(self, record): add_custom_log_levels() @@ -118,7 +121,7 @@ class Console_color_formatter(logging.Formatter): .value ) - return color_text(color, record.msg) + return color_text(color, super(Console_color_formatter, self).format(record)) def ansi_escape_code(color): # pragma: no cover @@ -179,6 +182,16 @@ def add_custom_log_levels(): # pragma: no cover add_logging_level('DISABLED', DISABLED) +def set_log_prefix(prefix): + ''' + Given a prefix string, set it onto the formatter defaults for every logging handler so that it + shows up in every subsequent logging message. For this to work, this relies on each logging + formatter to be initialized with "{prefix}" somewhere in its logging format. + ''' + for handler in logging.getLogger().handlers: + handler.formatter._style._defaults = {'prefix': f'{prefix}: ' if prefix else ''} + + def configure_logging( console_log_level, syslog_log_level=None, @@ -242,7 +255,7 @@ def configure_logging( if syslog_path: syslog_handler = logging.handlers.SysLogHandler(address=syslog_path) syslog_handler.setFormatter( - logging.Formatter('borgmatic: {levelname} {message}', style='{') # noqa: FS003 + logging.Formatter('borgmatic: {levelname} {prefix}{message}', style='{', defaults={'prefix': ''}) # noqa: FS003 ) syslog_handler.setLevel(syslog_log_level) handlers.append(syslog_handler) @@ -251,7 +264,7 @@ def configure_logging( file_handler = logging.handlers.WatchedFileHandler(log_file) file_handler.setFormatter( logging.Formatter( - log_file_format or '[{asctime}] {levelname}: {message}', style='{' # noqa: FS003 + log_file_format or '[{asctime}] {levelname}: {prefix}{message}', style='{', defaults={'prefix': ''} # noqa: FS003 ) ) file_handler.setLevel(log_file_log_level) diff --git a/pyproject.toml b/pyproject.toml index 2453e6b6..34f9c0d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "borgmatic" -version = "1.9.8" +version = "1.9.9.dev0" authors = [ { name="Dan Helfman", email="witten@torsion.org" }, ] diff --git a/tests/unit/actions/test_check.py b/tests/unit/actions/test_check.py index d5e0ea4a..90b716b5 100644 --- a/tests/unit/actions/test_check.py +++ b/tests/unit/actions/test_check.py @@ -931,7 +931,6 @@ def test_compare_spot_check_hashes_returns_paths_having_failing_hashes(): global_arguments=flexmock(), local_path=flexmock(), remote_path=flexmock(), - log_prefix='repo', source_paths=('/foo', '/bar', '/baz', '/quux'), ) == ('/bar',) @@ -972,7 +971,6 @@ def test_compare_spot_check_hashes_returns_relative_paths_having_failing_hashes( global_arguments=flexmock(), local_path=flexmock(), remote_path=flexmock(), - log_prefix='repo', source_paths=('foo', 'bar', 'baz', 'quux'), ) == ('bar',) @@ -1013,7 +1011,6 @@ def test_compare_spot_check_hashes_handles_data_sample_percentage_above_100(): global_arguments=flexmock(), local_path=flexmock(), remote_path=flexmock(), - log_prefix='repo', source_paths=('/foo', '/bar'), ) == ('/foo', '/bar') @@ -1051,7 +1048,6 @@ def test_compare_spot_check_hashes_uses_xxh64sum_command_option(): global_arguments=flexmock(), local_path=flexmock(), remote_path=flexmock(), - log_prefix='repo', source_paths=('/foo', '/bar', '/baz', '/quux'), ) == ('/bar',) @@ -1088,7 +1084,6 @@ def test_compare_spot_check_hashes_considers_path_missing_from_archive_as_not_ma global_arguments=flexmock(), local_path=flexmock(), remote_path=flexmock(), - log_prefix='repo', source_paths=('/foo', '/bar', '/baz', '/quux'), ) == ('/bar',) @@ -1124,7 +1119,6 @@ def test_compare_spot_check_hashes_considers_non_existent_path_as_not_matching() global_arguments=flexmock(), local_path=flexmock(), remote_path=flexmock(), - log_prefix='repo', source_paths=('/foo', '/bar', '/baz', '/quux'), ) == ('/bar',) @@ -1171,7 +1165,6 @@ def test_compare_spot_check_hashes_with_too_many_paths_feeds_them_to_commands_in global_arguments=flexmock(), local_path=flexmock(), remote_path=flexmock(), - log_prefix='repo', source_paths=('/foo', '/bar', '/baz', '/quux'), ) == ('/quux',) @@ -1214,7 +1207,6 @@ def test_compare_spot_check_hashes_uses_working_directory_to_access_source_paths global_arguments=flexmock(), local_path=flexmock(), remote_path=flexmock(), - log_prefix='repo', source_paths=('foo', 'bar', 'baz', 'quux'), ) == ('bar',) diff --git a/tests/unit/actions/test_restore.py b/tests/unit/actions/test_restore.py index 9debd701..6fedb367 100644 --- a/tests/unit/actions/test_restore.py +++ b/tests/unit/actions/test_restore.py @@ -194,7 +194,6 @@ def test_get_configured_data_source_matches_data_source_with_restore_dump(): 'postgresql_databases': [{'name': 'foo'}, {'name': 'bar'}], }, restore_dump=module.Dump('postgresql_databases', 'bar'), - log_prefix='test', ) == {'name': 'bar'} @@ -206,7 +205,6 @@ def test_get_configured_data_source_matches_nothing_when_nothing_configured(): module.get_configured_data_source( config={}, restore_dump=module.Dump('postgresql_databases', 'quux'), - log_prefix='test', ) is None ) @@ -222,7 +220,6 @@ def test_get_configured_data_source_matches_nothing_when_restore_dump_does_not_m 'postgresql_databases': [{'name': 'foo'}], }, restore_dump=module.Dump('postgresql_databases', 'quux'), - log_prefix='test', ) is None ) @@ -250,7 +247,6 @@ def test_get_configured_data_source_with_multiple_matching_data_sources_errors() ], }, restore_dump=module.Dump('postgresql_databases', 'bar'), - log_prefix='test', ) @@ -291,7 +287,6 @@ def test_restore_single_dump_extracts_and_restores_single_file_dump(): flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( function_name='restore_data_source_dump', config=object, - log_prefix=object, hook_name=object, data_source=object, dry_run=object, @@ -334,7 +329,6 @@ def test_restore_single_dump_extracts_and_restores_directory_dump(): flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( function_name='restore_data_source_dump', config=object, - log_prefix=object, hook_name=object, data_source=object, dry_run=object, @@ -377,7 +371,6 @@ def test_restore_single_dump_with_directory_dump_error_cleans_up_temporary_direc flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( function_name='restore_data_source_dump', config=object, - log_prefix=object, hook_name=object, data_source=object, dry_run=object, @@ -419,7 +412,6 @@ def test_restore_single_dump_with_directory_dump_and_dry_run_skips_directory_mov flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').with_args( function_name='restore_data_source_dump', config=object, - log_prefix=object, hook_name=object, data_source=object, dry_run=object, @@ -1064,17 +1056,14 @@ def test_run_restore_restores_data_source_configured_with_all_name(): flexmock(module).should_receive('get_configured_data_source').with_args( config=object, restore_dump=module.Dump(hook_name='postgresql_databases', data_source_name='foo'), - log_prefix=object, ).and_return({'name': 'foo'}) flexmock(module).should_receive('get_configured_data_source').with_args( config=object, restore_dump=module.Dump(hook_name='postgresql_databases', data_source_name='bar'), - log_prefix=object, ).and_return(None) flexmock(module).should_receive('get_configured_data_source').with_args( config=object, restore_dump=module.Dump(hook_name='postgresql_databases', data_source_name='all'), - log_prefix=object, ).and_return({'name': 'bar'}) flexmock(module).should_receive('restore_single_dump').with_args( repository=object, @@ -1148,17 +1137,14 @@ def test_run_restore_skips_missing_data_source(): flexmock(module).should_receive('get_configured_data_source').with_args( config=object, restore_dump=module.Dump(hook_name='postgresql_databases', data_source_name='foo'), - log_prefix=object, ).and_return({'name': 'foo'}) flexmock(module).should_receive('get_configured_data_source').with_args( config=object, restore_dump=module.Dump(hook_name='postgresql_databases', data_source_name='bar'), - log_prefix=object, ).and_return(None) flexmock(module).should_receive('get_configured_data_source').with_args( config=object, restore_dump=module.Dump(hook_name='postgresql_databases', data_source_name='all'), - log_prefix=object, ).and_return(None) flexmock(module).should_receive('restore_single_dump').with_args( repository=object, @@ -1232,12 +1218,10 @@ def test_run_restore_restores_data_sources_from_different_hooks(): flexmock(module).should_receive('get_configured_data_source').with_args( config=object, restore_dump=module.Dump(hook_name='postgresql_databases', data_source_name='foo'), - log_prefix=object, ).and_return({'name': 'foo'}) flexmock(module).should_receive('get_configured_data_source').with_args( config=object, restore_dump=module.Dump(hook_name='mysql_databases', data_source_name='foo'), - log_prefix=object, ).and_return({'name': 'bar'}) flexmock(module).should_receive('restore_single_dump').with_args( repository=object, diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 5e3368e9..84abcddc 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -17,13 +17,12 @@ def test_write_patterns_file_writes_pattern_lines(): module.write_patterns_file( [Pattern('/foo'), Pattern('/foo/bar', Pattern_type.INCLUDE, Pattern_style.SHELL)], borgmatic_runtime_directory='/run/user/0', - log_prefix='test.yaml', ) def test_write_patterns_file_with_empty_exclude_patterns_does_not_raise(): module.write_patterns_file( - [], borgmatic_runtime_directory='/run/user/0', log_prefix='test.yaml' + [], borgmatic_runtime_directory='/run/user/0' ) @@ -36,7 +35,6 @@ def test_write_patterns_file_appends_to_existing(): module.write_patterns_file( [Pattern('/foo'), Pattern('/foo/bar', Pattern_type.INCLUDE)], borgmatic_runtime_directory='/run/user/0', - log_prefix='test.yaml', patterns_file=patterns_file, ) diff --git a/tests/unit/hooks/data_source/test_bootstrap.py b/tests/unit/hooks/data_source/test_bootstrap.py index a1710dd7..1623dfb7 100644 --- a/tests/unit/hooks/data_source/test_bootstrap.py +++ b/tests/unit/hooks/data_source/test_bootstrap.py @@ -24,7 +24,6 @@ def test_dump_data_sources_creates_manifest_file(): module.dump_data_sources( hook_config=None, config={}, - log_prefix='test', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -40,7 +39,6 @@ def test_dump_data_sources_with_store_config_files_false_does_not_create_manifes module.dump_data_sources( hook_config=hook_config, config={'bootstrap': hook_config}, - log_prefix='test', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -55,7 +53,6 @@ def test_dump_data_sources_with_dry_run_does_not_create_manifest_file(): module.dump_data_sources( hook_config=None, config={}, - log_prefix='test', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -76,7 +73,6 @@ def test_remove_data_source_dumps_deletes_manifest_and_parent_directory(): module.remove_data_source_dumps( hook_config=None, config={}, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -93,7 +89,6 @@ def test_remove_data_source_dumps_with_dry_run_bails(): module.remove_data_source_dumps( hook_config=None, config={}, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=True, ) @@ -112,7 +107,6 @@ def test_remove_data_source_dumps_swallows_manifest_file_not_found_error(): module.remove_data_source_dumps( hook_config=None, config={}, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -133,7 +127,6 @@ def test_remove_data_source_dumps_swallows_manifest_parent_directory_not_found_e module.remove_data_source_dumps( hook_config=None, config={}, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) diff --git a/tests/unit/hooks/data_source/test_btrfs.py b/tests/unit/hooks/data_source/test_btrfs.py index 72f77f82..c669a996 100644 --- a/tests/unit/hooks/data_source/test_btrfs.py +++ b/tests/unit/hooks/data_source/test_btrfs.py @@ -242,7 +242,6 @@ def test_dump_data_sources_snapshots_each_subvolume_and_updates_patterns(): module.dump_data_sources( hook_config=config['btrfs'], config=config, - log_prefix='test', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=patterns, @@ -300,7 +299,6 @@ def test_dump_data_sources_uses_custom_btrfs_command_in_commands(): module.dump_data_sources( hook_config=config['btrfs'], config=config, - log_prefix='test', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=patterns, @@ -356,7 +354,6 @@ def test_dump_data_sources_uses_custom_findmnt_command_in_commands(): module.dump_data_sources( hook_config=config['btrfs'], config=config, - log_prefix='test', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=patterns, @@ -397,7 +394,6 @@ def test_dump_data_sources_with_dry_run_skips_snapshot_and_patterns_update(): module.dump_data_sources( hook_config=config['btrfs'], config=config, - log_prefix='test', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=patterns, @@ -422,7 +418,6 @@ def test_dump_data_sources_without_matching_subvolumes_skips_snapshot_and_patter module.dump_data_sources( hook_config=config['btrfs'], config=config, - log_prefix='test', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=patterns, @@ -485,7 +480,6 @@ def test_dump_data_sources_snapshots_adds_to_existing_exclude_patterns(): module.dump_data_sources( hook_config=config['btrfs'], config=config, - log_prefix='test', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=patterns, @@ -595,7 +589,6 @@ def test_remove_data_source_dumps_deletes_snapshots(): module.remove_data_source_dumps( hook_config=config['btrfs'], config=config, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -613,7 +606,6 @@ def test_remove_data_source_dumps_without_hook_configuration_bails(): module.remove_data_source_dumps( hook_config=None, config={'source_directories': '/mnt/subvolume'}, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -632,7 +624,6 @@ def test_remove_data_source_dumps_with_get_subvolumes_file_not_found_error_bails module.remove_data_source_dumps( hook_config=config['btrfs'], config=config, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -653,7 +644,6 @@ def test_remove_data_source_dumps_with_get_subvolumes_called_process_error_bails module.remove_data_source_dumps( hook_config=config['btrfs'], config=config, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -717,7 +707,6 @@ def test_remove_data_source_dumps_with_dry_run_skips_deletes(): module.remove_data_source_dumps( hook_config=config['btrfs'], config=config, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=True, ) @@ -736,7 +725,6 @@ def test_remove_data_source_dumps_without_subvolumes_skips_deletes(): module.remove_data_source_dumps( hook_config=config['btrfs'], config=config, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -780,7 +768,6 @@ def test_remove_data_source_without_snapshots_skips_deletes(): module.remove_data_source_dumps( hook_config=config['btrfs'], config=config, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -844,7 +831,6 @@ def test_remove_data_source_dumps_with_delete_snapshot_file_not_found_error_bail module.remove_data_source_dumps( hook_config=config['btrfs'], config=config, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -910,7 +896,6 @@ def test_remove_data_source_dumps_with_delete_snapshot_called_process_error_bail module.remove_data_source_dumps( hook_config=config['btrfs'], config=config, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) diff --git a/tests/unit/hooks/data_source/test_lvm.py b/tests/unit/hooks/data_source/test_lvm.py index 6b06cefc..49159905 100644 --- a/tests/unit/hooks/data_source/test_lvm.py +++ b/tests/unit/hooks/data_source/test_lvm.py @@ -220,7 +220,6 @@ def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns(): module.dump_data_sources( hook_config=config['lvm'], config=config, - log_prefix='test', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=patterns, @@ -246,7 +245,6 @@ def test_dump_data_sources_with_no_logical_volumes_skips_snapshots(): module.dump_data_sources( hook_config=config['lvm'], config=config, - log_prefix='test', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=patterns, @@ -317,7 +315,6 @@ def test_dump_data_sources_uses_snapshot_size_for_snapshot(): module.dump_data_sources( hook_config=config['lvm'], config=config, - log_prefix='test', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=patterns, @@ -398,7 +395,6 @@ def test_dump_data_sources_uses_custom_commands(): module.dump_data_sources( hook_config=config['lvm'], config=config, - log_prefix='test', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=patterns, @@ -450,7 +446,6 @@ def test_dump_data_sources_with_dry_run_skips_snapshots_and_does_not_touch_patte module.dump_data_sources( hook_config=config['lvm'], config=config, - log_prefix='test', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=patterns, @@ -518,7 +513,6 @@ def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained module.dump_data_sources( hook_config=config['lvm'], config=config, - log_prefix='test', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=patterns, @@ -572,7 +566,6 @@ def test_dump_data_sources_with_missing_snapshot_errors(): module.dump_data_sources( hook_config=config['lvm'], config=config, - log_prefix='test', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=patterns, @@ -728,7 +721,6 @@ def test_remove_data_source_dumps_unmounts_and_remove_snapshots(): module.remove_data_source_dumps( hook_config=config['lvm'], config=config, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -745,7 +737,6 @@ def test_remove_data_source_dumps_bails_for_missing_lvm_configuration(): module.remove_data_source_dumps( hook_config=None, config={'source_directories': '/mnt/lvolume'}, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -763,7 +754,6 @@ def test_remove_data_source_dumps_bails_for_missing_lsblk_command(): module.remove_data_source_dumps( hook_config=config['lvm'], config=config, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -783,7 +773,6 @@ def test_remove_data_source_dumps_bails_for_lsblk_command_error(): module.remove_data_source_dumps( hook_config=config['lvm'], config=config, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -828,7 +817,6 @@ def test_remove_data_source_dumps_with_missing_snapshot_directory_skips_unmount( module.remove_data_source_dumps( hook_config=config['lvm'], config=config, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -886,7 +874,6 @@ def test_remove_data_source_dumps_with_missing_snapshot_mount_path_skips_unmount module.remove_data_source_dumps( hook_config=config['lvm'], config=config, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -944,7 +931,6 @@ def test_remove_data_source_dumps_with_successful_mount_point_removal_skips_unmo module.remove_data_source_dumps( hook_config=config['lvm'], config=config, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -988,7 +974,6 @@ def test_remove_data_source_dumps_bails_for_missing_umount_command(): module.remove_data_source_dumps( hook_config=config['lvm'], config=config, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -1032,7 +1017,6 @@ def test_remove_data_source_dumps_bails_for_umount_command_error(): module.remove_data_source_dumps( hook_config=config['lvm'], config=config, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -1076,7 +1060,6 @@ def test_remove_data_source_dumps_bails_for_missing_lvs_command(): module.remove_data_source_dumps( hook_config=config['lvm'], config=config, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -1122,7 +1105,6 @@ def test_remove_data_source_dumps_bails_for_lvs_command_error(): module.remove_data_source_dumps( hook_config=config['lvm'], config=config, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -1165,7 +1147,6 @@ def test_remove_data_source_with_dry_run_skips_snapshot_unmount_and_delete(): module.remove_data_source_dumps( hook_config=config['lvm'], config=config, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=True, ) diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py index 62e6707a..cc711738 100644 --- a/tests/unit/hooks/data_source/test_mariadb.py +++ b/tests/unit/hooks/data_source/test_mariadb.py @@ -8,10 +8,9 @@ from borgmatic.hooks.data_source import mariadb as module def test_database_names_to_dump_passes_through_name(): extra_environment = flexmock() - log_prefix = '' names = module.database_names_to_dump( - {'name': 'foo'}, extra_environment, log_prefix, dry_run=False + {'name': 'foo'}, extra_environment, dry_run=False ) assert names == ('foo',) @@ -19,11 +18,10 @@ def test_database_names_to_dump_passes_through_name(): def test_database_names_to_dump_bails_for_dry_run(): extra_environment = flexmock() - log_prefix = '' flexmock(module).should_receive('execute_command_and_capture_output').never() names = module.database_names_to_dump( - {'name': 'all'}, extra_environment, log_prefix, dry_run=True + {'name': 'all'}, extra_environment, dry_run=True ) assert names == () @@ -31,14 +29,13 @@ def test_database_names_to_dump_bails_for_dry_run(): def test_database_names_to_dump_queries_mariadb_for_database_names(): extra_environment = flexmock() - log_prefix = '' flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('mariadb', '--skip-column-names', '--batch', '--execute', 'show schemas'), extra_environment=extra_environment, ).and_return('foo\nbar\nmysql\n').once() names = module.database_names_to_dump( - {'name': 'all'}, extra_environment, log_prefix, dry_run=False + {'name': 'all'}, extra_environment, dry_run=False ) assert names == ('foo', 'bar') @@ -46,12 +43,12 @@ def test_database_names_to_dump_queries_mariadb_for_database_names(): def test_use_streaming_true_for_any_databases(): assert module.use_streaming( - databases=[flexmock(), flexmock()], config=flexmock(), log_prefix=flexmock() + databases=[flexmock(), flexmock()], config=flexmock(), ) def test_use_streaming_false_for_no_databases(): - assert not module.use_streaming(databases=[], config=flexmock(), log_prefix=flexmock()) + assert not module.use_streaming(databases=[], config=flexmock()) def test_dump_data_sources_dumps_each_database(): @@ -65,7 +62,6 @@ def test_dump_data_sources_dumps_each_database(): for name, process in zip(('foo', 'bar'), processes): flexmock(module).should_receive('execute_dump_command').with_args( database={'name': name}, - log_prefix=object, dump_path=object, database_names=(name,), extra_environment=object, @@ -97,7 +93,6 @@ def test_dump_data_sources_dumps_with_password(): flexmock(module).should_receive('execute_dump_command').with_args( database=database, - log_prefix=object, dump_path=object, database_names=('foo',), extra_environment={'MYSQL_PWD': 'trustsome1'}, @@ -123,7 +118,6 @@ def test_dump_data_sources_dumps_all_databases_at_once(): flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) flexmock(module).should_receive('execute_dump_command').with_args( database={'name': 'all'}, - log_prefix=object, dump_path=object, database_names=('foo', 'bar'), extra_environment=object, @@ -151,7 +145,6 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured for name, process in zip(('foo', 'bar'), processes): flexmock(module).should_receive('execute_dump_command').with_args( database={'name': name, 'format': 'sql'}, - log_prefix=object, dump_path=object, database_names=(name,), extra_environment=object, @@ -233,7 +226,6 @@ def test_execute_dump_command_runs_mariadb_dump(): assert ( module.execute_dump_command( database={'name': 'foo'}, - log_prefix='log', dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -265,7 +257,6 @@ def test_execute_dump_command_runs_mariadb_dump_without_add_drop_database(): assert ( module.execute_dump_command( database={'name': 'foo', 'add_drop_database': False}, - log_prefix='log', dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -304,7 +295,6 @@ def test_execute_dump_command_runs_mariadb_dump_with_hostname_and_port(): assert ( module.execute_dump_command( database={'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}, - log_prefix='log', dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -339,7 +329,6 @@ def test_execute_dump_command_runs_mariadb_dump_with_username_and_password(): assert ( module.execute_dump_command( database={'name': 'foo', 'username': 'root', 'password': 'trustsome1'}, - log_prefix='log', dump_path=flexmock(), database_names=('foo',), extra_environment={'MYSQL_PWD': 'trustsome1'}, @@ -373,7 +362,6 @@ def test_execute_dump_command_runs_mariadb_dump_with_options(): assert ( module.execute_dump_command( database={'name': 'foo', 'options': '--stuff=such'}, - log_prefix='log', dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -411,7 +399,6 @@ def test_execute_dump_command_runs_non_default_mariadb_dump_with_options(): 'mariadb_dump_command': 'custom_mariadb_dump', 'options': '--stuff=such', }, # Custom MariaDB dump command specified - log_prefix='log', dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -431,7 +418,6 @@ def test_execute_dump_command_with_duplicate_dump_skips_mariadb_dump(): assert ( module.execute_dump_command( database={'name': 'foo'}, - log_prefix='log', dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -452,7 +438,6 @@ def test_execute_dump_command_with_dry_run_skips_mariadb_dump(): assert ( module.execute_dump_command( database={'name': 'foo'}, - log_prefix='log', dump_path=flexmock(), database_names=('foo',), extra_environment=None, diff --git a/tests/unit/hooks/data_source/test_mongodb.py b/tests/unit/hooks/data_source/test_mongodb.py index 1923afc9..0e3efd5d 100644 --- a/tests/unit/hooks/data_source/test_mongodb.py +++ b/tests/unit/hooks/data_source/test_mongodb.py @@ -9,7 +9,6 @@ def test_use_streaming_true_for_any_non_directory_format_databases(): assert module.use_streaming( databases=[{'format': 'stuff'}, {'format': 'directory'}, {}], config=flexmock(), - log_prefix=flexmock(), ) @@ -17,12 +16,11 @@ def test_use_streaming_false_for_all_directory_format_databases(): assert not module.use_streaming( databases=[{'format': 'directory'}, {'format': 'directory'}], config=flexmock(), - log_prefix=flexmock(), ) def test_use_streaming_false_for_no_databases(): - assert not module.use_streaming(databases=[], config=flexmock(), log_prefix=flexmock()) + assert not module.use_streaming(databases=[], config=flexmock()) def test_dump_data_sources_runs_mongodump_for_each_database(): diff --git a/tests/unit/hooks/data_source/test_mysql.py b/tests/unit/hooks/data_source/test_mysql.py index 3f450a4c..b9ab1dfb 100644 --- a/tests/unit/hooks/data_source/test_mysql.py +++ b/tests/unit/hooks/data_source/test_mysql.py @@ -8,10 +8,9 @@ from borgmatic.hooks.data_source import mysql as module def test_database_names_to_dump_passes_through_name(): extra_environment = flexmock() - log_prefix = '' names = module.database_names_to_dump( - {'name': 'foo'}, extra_environment, log_prefix, dry_run=False + {'name': 'foo'}, extra_environment, dry_run=False ) assert names == ('foo',) @@ -19,11 +18,10 @@ def test_database_names_to_dump_passes_through_name(): def test_database_names_to_dump_bails_for_dry_run(): extra_environment = flexmock() - log_prefix = '' flexmock(module).should_receive('execute_command_and_capture_output').never() names = module.database_names_to_dump( - {'name': 'all'}, extra_environment, log_prefix, dry_run=True + {'name': 'all'}, extra_environment, dry_run=True ) assert names == () @@ -31,14 +29,13 @@ def test_database_names_to_dump_bails_for_dry_run(): def test_database_names_to_dump_queries_mysql_for_database_names(): extra_environment = flexmock() - log_prefix = '' flexmock(module).should_receive('execute_command_and_capture_output').with_args( ('mysql', '--skip-column-names', '--batch', '--execute', 'show schemas'), extra_environment=extra_environment, ).and_return('foo\nbar\nmysql\n').once() names = module.database_names_to_dump( - {'name': 'all'}, extra_environment, log_prefix, dry_run=False + {'name': 'all'}, extra_environment, dry_run=False ) assert names == ('foo', 'bar') @@ -46,12 +43,12 @@ def test_database_names_to_dump_queries_mysql_for_database_names(): def test_use_streaming_true_for_any_databases(): assert module.use_streaming( - databases=[flexmock(), flexmock()], config=flexmock(), log_prefix=flexmock() + databases=[flexmock(), flexmock()], config=flexmock(), ) def test_use_streaming_false_for_no_databases(): - assert not module.use_streaming(databases=[], config=flexmock(), log_prefix=flexmock()) + assert not module.use_streaming(databases=[], config=flexmock()) def test_dump_data_sources_dumps_each_database(): @@ -65,7 +62,6 @@ def test_dump_data_sources_dumps_each_database(): for name, process in zip(('foo', 'bar'), processes): flexmock(module).should_receive('execute_dump_command').with_args( database={'name': name}, - log_prefix=object, dump_path=object, database_names=(name,), extra_environment=object, @@ -97,7 +93,6 @@ def test_dump_data_sources_dumps_with_password(): flexmock(module).should_receive('execute_dump_command').with_args( database=database, - log_prefix=object, dump_path=object, database_names=('foo',), extra_environment={'MYSQL_PWD': 'trustsome1'}, @@ -123,7 +118,6 @@ def test_dump_data_sources_dumps_all_databases_at_once(): flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar')) flexmock(module).should_receive('execute_dump_command').with_args( database={'name': 'all'}, - log_prefix=object, dump_path=object, database_names=('foo', 'bar'), extra_environment=object, @@ -151,7 +145,6 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured for name, process in zip(('foo', 'bar'), processes): flexmock(module).should_receive('execute_dump_command').with_args( database={'name': name, 'format': 'sql'}, - log_prefix=object, dump_path=object, database_names=(name,), extra_environment=object, @@ -233,7 +226,6 @@ def test_execute_dump_command_runs_mysqldump(): assert ( module.execute_dump_command( database={'name': 'foo'}, - log_prefix='log', dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -265,7 +257,6 @@ def test_execute_dump_command_runs_mysqldump_without_add_drop_database(): assert ( module.execute_dump_command( database={'name': 'foo', 'add_drop_database': False}, - log_prefix='log', dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -304,7 +295,6 @@ def test_execute_dump_command_runs_mysqldump_with_hostname_and_port(): assert ( module.execute_dump_command( database={'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}, - log_prefix='log', dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -339,7 +329,6 @@ def test_execute_dump_command_runs_mysqldump_with_username_and_password(): assert ( module.execute_dump_command( database={'name': 'foo', 'username': 'root', 'password': 'trustsome1'}, - log_prefix='log', dump_path=flexmock(), database_names=('foo',), extra_environment={'MYSQL_PWD': 'trustsome1'}, @@ -373,7 +362,6 @@ def test_execute_dump_command_runs_mysqldump_with_options(): assert ( module.execute_dump_command( database={'name': 'foo', 'options': '--stuff=such'}, - log_prefix='log', dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -409,7 +397,6 @@ def test_execute_dump_command_runs_non_default_mysqldump(): 'name': 'foo', 'mysql_dump_command': 'custom_mysqldump', }, # Custom MySQL dump command specified - log_prefix='log', dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -429,7 +416,6 @@ def test_execute_dump_command_with_duplicate_dump_skips_mysqldump(): assert ( module.execute_dump_command( database={'name': 'foo'}, - log_prefix='log', dump_path=flexmock(), database_names=('foo',), extra_environment=None, @@ -450,7 +436,6 @@ def test_execute_dump_command_with_dry_run_skips_mysqldump(): assert ( module.execute_dump_command( database={'name': 'foo'}, - log_prefix='log', dump_path=flexmock(), database_names=('foo',), extra_environment=None, diff --git a/tests/unit/hooks/data_source/test_postgresql.py b/tests/unit/hooks/data_source/test_postgresql.py index 45c0e80a..752f940b 100644 --- a/tests/unit/hooks/data_source/test_postgresql.py +++ b/tests/unit/hooks/data_source/test_postgresql.py @@ -203,7 +203,6 @@ def test_use_streaming_true_for_any_non_directory_format_databases(): assert module.use_streaming( databases=[{'format': 'stuff'}, {'format': 'directory'}, {}], config=flexmock(), - log_prefix=flexmock(), ) @@ -211,12 +210,11 @@ def test_use_streaming_false_for_all_directory_format_databases(): assert not module.use_streaming( databases=[{'format': 'directory'}, {'format': 'directory'}], config=flexmock(), - log_prefix=flexmock(), ) def test_use_streaming_false_for_no_databases(): - assert not module.use_streaming(databases=[], config=flexmock(), log_prefix=flexmock()) + assert not module.use_streaming(databases=[], config=flexmock()) def test_dump_data_sources_runs_pg_dump_for_each_database(): diff --git a/tests/unit/hooks/data_source/test_sqlite.py b/tests/unit/hooks/data_source/test_sqlite.py index f7acce8d..41526793 100644 --- a/tests/unit/hooks/data_source/test_sqlite.py +++ b/tests/unit/hooks/data_source/test_sqlite.py @@ -7,12 +7,12 @@ from borgmatic.hooks.data_source import sqlite as module def test_use_streaming_true_for_any_databases(): assert module.use_streaming( - databases=[flexmock(), flexmock()], config=flexmock(), log_prefix=flexmock() + databases=[flexmock(), flexmock()], config=flexmock(), ) def test_use_streaming_false_for_no_databases(): - assert not module.use_streaming(databases=[], config=flexmock(), log_prefix=flexmock()) + assert not module.use_streaming(databases=[], config=flexmock()) def test_dump_data_sources_logs_and_skips_if_dump_already_exists(): diff --git a/tests/unit/hooks/data_source/test_zfs.py b/tests/unit/hooks/data_source/test_zfs.py index 7700b736..85782bdb 100644 --- a/tests/unit/hooks/data_source/test_zfs.py +++ b/tests/unit/hooks/data_source/test_zfs.py @@ -154,7 +154,6 @@ def test_dump_data_sources_snapshots_and_mounts_and_updates_patterns(): module.dump_data_sources( hook_config={}, config={'source_directories': '/mnt/dataset', 'zfs': {}}, - log_prefix='test', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=patterns, @@ -177,7 +176,6 @@ def test_dump_data_sources_with_no_datasets_skips_snapshots(): module.dump_data_sources( hook_config={}, config={'patterns': flexmock(), 'zfs': {}}, - log_prefix='test', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=patterns, @@ -227,7 +225,6 @@ def test_dump_data_sources_uses_custom_commands(): 'patterns': flexmock(), 'zfs': hook_config, }, - log_prefix='test', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=patterns, @@ -252,7 +249,6 @@ def test_dump_data_sources_with_dry_run_skips_commands_and_does_not_touch_patter module.dump_data_sources( hook_config={}, config={'patterns': ('R /mnt/dataset',), 'zfs': {}}, - log_prefix='test', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=patterns, @@ -295,7 +291,6 @@ def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained module.dump_data_sources( hook_config={}, config={'patterns': ('R /mnt/dataset',), 'zfs': {}}, - log_prefix='test', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=patterns, @@ -338,7 +333,6 @@ def test_remove_data_source_dumps_unmounts_and_destroys_snapshots(): module.remove_data_source_dumps( hook_config={}, config={'source_directories': '/mnt/dataset', 'zfs': {}}, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -366,7 +360,6 @@ def test_remove_data_source_dumps_use_custom_commands(): module.remove_data_source_dumps( hook_config=hook_config, config={'source_directories': '/mnt/dataset', 'zfs': hook_config}, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -381,7 +374,6 @@ def test_remove_data_source_dumps_bails_for_missing_hook_configuration(): module.remove_data_source_dumps( hook_config=None, config={'source_directories': '/mnt/dataset'}, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -397,7 +389,6 @@ def test_remove_data_source_dumps_bails_for_missing_zfs_command(): module.remove_data_source_dumps( hook_config=hook_config, config={'source_directories': '/mnt/dataset', 'zfs': hook_config}, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -415,7 +406,6 @@ def test_remove_data_source_dumps_bails_for_zfs_command_error(): module.remove_data_source_dumps( hook_config=hook_config, config={'source_directories': '/mnt/dataset', 'zfs': hook_config}, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -439,7 +429,6 @@ def test_remove_data_source_dumps_bails_for_missing_umount_command(): module.remove_data_source_dumps( hook_config=hook_config, config={'source_directories': '/mnt/dataset', 'zfs': hook_config}, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -463,7 +452,6 @@ def test_remove_data_source_dumps_bails_for_umount_command_error(): module.remove_data_source_dumps( hook_config=hook_config, config={'source_directories': '/mnt/dataset', 'zfs': hook_config}, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -488,7 +476,6 @@ def test_remove_data_source_dumps_skips_unmount_snapshot_directories_that_are_no module.remove_data_source_dumps( hook_config={}, config={'source_directories': '/mnt/dataset', 'zfs': {}}, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -518,7 +505,6 @@ def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_that_are_no module.remove_data_source_dumps( hook_config={}, config={'source_directories': '/mnt/dataset', 'zfs': {}}, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -548,7 +534,6 @@ def test_remove_data_source_dumps_skips_unmount_snapshot_mount_paths_after_rmtre module.remove_data_source_dumps( hook_config={}, config={'source_directories': '/mnt/dataset', 'zfs': {}}, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=False, ) @@ -571,7 +556,6 @@ def test_remove_data_source_dumps_with_dry_run_skips_unmount_and_destroy(): module.remove_data_source_dumps( hook_config={}, config={'source_directories': '/mnt/dataset', 'zfs': {}}, - log_prefix='test', borgmatic_runtime_directory='/run/borgmatic', dry_run=True, ) diff --git a/tests/unit/hooks/test_command.py b/tests/unit/hooks/test_command.py index 4a3ca045..01d04be0 100644 --- a/tests/unit/hooks/test_command.py +++ b/tests/unit/hooks/test_command.py @@ -7,13 +7,13 @@ from borgmatic.hooks import command as module def test_interpolate_context_passes_through_command_without_variable(): - assert module.interpolate_context('test.yaml', 'pre-backup', 'ls', {'foo': 'bar'}) == 'ls' + assert module.interpolate_context('pre-backup', 'ls', {'foo': 'bar'}) == 'ls' def test_interpolate_context_passes_through_command_with_unknown_variable(): command = 'ls {baz}' # noqa: FS003 - assert module.interpolate_context('test.yaml', 'pre-backup', command, {'foo': 'bar'}) == command + assert module.interpolate_context('pre-backup', command, {'foo': 'bar'}) == command def test_interpolate_context_interpolates_variables(): @@ -21,7 +21,7 @@ def test_interpolate_context_interpolates_variables(): context = {'foo': 'bar', 'baz': 'quux'} assert ( - module.interpolate_context('test.yaml', 'pre-backup', command, context) == 'ls barquux quux' + module.interpolate_context('pre-backup', command, context) == 'ls barquux quux' ) @@ -30,7 +30,7 @@ def test_interpolate_context_escapes_interpolated_variables(): context = {'foo': 'bar', 'inject': 'hi; naughty-command'} assert ( - module.interpolate_context('test.yaml', 'pre-backup', command, context) + module.interpolate_context('pre-backup', command, context) == "ls bar 'hi; naughty-command'" ) @@ -53,7 +53,7 @@ def test_make_environment_with_pyinstaller_and_LD_LIBRARY_PATH_ORIG_copies_it_in def test_execute_hook_invokes_each_command(): flexmock(module).should_receive('interpolate_context').replace_with( - lambda config_file, hook_description, command, context: command + lambda hook_description, command, context: command ) flexmock(module).should_receive('make_environment').and_return({}) flexmock(module.borgmatic.execute).should_receive('execute_command').with_args( @@ -68,7 +68,7 @@ def test_execute_hook_invokes_each_command(): def test_execute_hook_with_multiple_commands_invokes_each_command(): flexmock(module).should_receive('interpolate_context').replace_with( - lambda config_file, hook_description, command, context: command + lambda hook_description, command, context: command ) flexmock(module).should_receive('make_environment').and_return({}) flexmock(module.borgmatic.execute).should_receive('execute_command').with_args( @@ -89,7 +89,7 @@ def test_execute_hook_with_multiple_commands_invokes_each_command(): def test_execute_hook_with_umask_sets_that_umask(): flexmock(module).should_receive('interpolate_context').replace_with( - lambda config_file, hook_description, command, context: command + lambda hook_description, command, context: command ) flexmock(module.os).should_receive('umask').with_args(0o77).and_return(0o22).once() flexmock(module.os).should_receive('umask').with_args(0o22).once() @@ -106,7 +106,7 @@ def test_execute_hook_with_umask_sets_that_umask(): def test_execute_hook_with_dry_run_skips_commands(): flexmock(module).should_receive('interpolate_context').replace_with( - lambda config_file, hook_description, command, context: command + lambda hook_description, command, context: command ) flexmock(module).should_receive('make_environment').and_return({}) flexmock(module.borgmatic.execute).should_receive('execute_command').never() @@ -120,7 +120,7 @@ def test_execute_hook_with_empty_commands_does_not_raise(): def test_execute_hook_on_error_logs_as_error(): flexmock(module).should_receive('interpolate_context').replace_with( - lambda config_file, hook_description, command, context: command + lambda hook_description, command, context: command ) flexmock(module).should_receive('make_environment').and_return({}) flexmock(module.borgmatic.execute).should_receive('execute_command').with_args( @@ -136,14 +136,14 @@ def test_execute_hook_on_error_logs_as_error(): def test_considered_soft_failure_treats_soft_fail_exit_code_as_soft_fail(): error = subprocess.CalledProcessError(module.SOFT_FAIL_EXIT_CODE, 'try again') - assert module.considered_soft_failure('config.yaml', error) + assert module.considered_soft_failure(error) def test_considered_soft_failure_does_not_treat_other_exit_code_as_soft_fail(): error = subprocess.CalledProcessError(1, 'error') - assert not module.considered_soft_failure('config.yaml', error) + assert not module.considered_soft_failure(error) def test_considered_soft_failure_does_not_treat_other_exception_type_as_soft_fail(): - assert not module.considered_soft_failure('config.yaml', Exception()) + assert not module.considered_soft_failure(Exception()) diff --git a/tests/unit/hooks/test_dispatch.py b/tests/unit/hooks/test_dispatch.py index 619b4366..dde8bd5d 100644 --- a/tests/unit/hooks/test_dispatch.py +++ b/tests/unit/hooks/test_dispatch.py @@ -6,7 +6,7 @@ from flexmock import flexmock from borgmatic.hooks import dispatch as module -def hook_function(hook_config, config, log_prefix, thing, value): +def hook_function(hook_config, config, thing, value): ''' This test function gets mocked out below. ''' From 8c5db194905ee4e1d58a5ff69b0e3768af6e9b63 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 25 Jan 2025 14:14:48 -0800 Subject: [PATCH 02/12] Code formatting (#635). --- borgmatic/actions/borg.py | 4 +-- borgmatic/actions/break_lock.py | 4 +-- borgmatic/actions/change_passphrase.py | 4 +-- borgmatic/actions/check.py | 4 +-- borgmatic/actions/compact.py | 8 ++---- borgmatic/actions/create.py | 4 +-- borgmatic/actions/export_tar.py | 4 +-- borgmatic/actions/extract.py | 4 +-- borgmatic/actions/info.py | 4 +-- borgmatic/actions/mount.py | 4 +-- borgmatic/actions/repo_delete.py | 3 +- borgmatic/actions/repo_info.py | 4 +-- borgmatic/actions/restore.py | 8 ++---- borgmatic/actions/transfer.py | 4 +-- borgmatic/commands/borgmatic.py | 30 +++++++++++--------- borgmatic/hooks/command.py | 4 +-- borgmatic/hooks/data_source/bootstrap.py | 4 +-- borgmatic/hooks/data_source/lvm.py | 8 ++---- borgmatic/hooks/data_source/mariadb.py | 12 ++------ borgmatic/hooks/data_source/mongodb.py | 4 +-- borgmatic/hooks/data_source/mysql.py | 12 ++------ borgmatic/hooks/data_source/postgresql.py | 8 ++---- borgmatic/hooks/data_source/sqlite.py | 4 +-- borgmatic/hooks/data_source/zfs.py | 8 ++---- borgmatic/logger.py | 16 ++++++++--- tests/unit/borg/test_create.py | 4 +-- tests/unit/hooks/data_source/test_mariadb.py | 15 ++++------ tests/unit/hooks/data_source/test_mysql.py | 15 ++++------ tests/unit/hooks/data_source/test_sqlite.py | 3 +- tests/unit/hooks/test_command.py | 7 ++--- 30 files changed, 75 insertions(+), 142 deletions(-) diff --git a/borgmatic/actions/borg.py b/borgmatic/actions/borg.py index fe2facdb..cb9f6e3f 100644 --- a/borgmatic/actions/borg.py +++ b/borgmatic/actions/borg.py @@ -22,9 +22,7 @@ def run_borg( if borg_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, borg_arguments.repository ): - logger.info( - 'Running arbitrary Borg command' - ) + logger.info('Running arbitrary Borg command') archive_name = borgmatic.borg.repo_list.resolve_archive_name( repository['path'], borg_arguments.archive, diff --git a/borgmatic/actions/break_lock.py b/borgmatic/actions/break_lock.py index 8375a6eb..80319a30 100644 --- a/borgmatic/actions/break_lock.py +++ b/borgmatic/actions/break_lock.py @@ -21,9 +21,7 @@ def run_break_lock( if break_lock_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, break_lock_arguments.repository ): - logger.info( - 'Breaking repository and cache locks' - ) + logger.info('Breaking repository and cache locks') borgmatic.borg.break_lock.break_lock( repository['path'], config, diff --git a/borgmatic/actions/change_passphrase.py b/borgmatic/actions/change_passphrase.py index 3671d72c..7c4a3974 100644 --- a/borgmatic/actions/change_passphrase.py +++ b/borgmatic/actions/change_passphrase.py @@ -24,9 +24,7 @@ def run_change_passphrase( repository, change_passphrase_arguments.repository ) ): - logger.info( - 'Changing repository passphrase' - ) + logger.info('Changing repository passphrase') borgmatic.borg.change_passphrase.change_passphrase( repository['path'], config, diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index 2555613f..2d514ad0 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -762,9 +762,7 @@ def run_check( write_check_time(make_check_time_path(config, repository_id, 'extract')) if 'spot' in checks: - with borgmatic.config.paths.Runtime_directory( - config - ) as borgmatic_runtime_directory: + with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory: spot_check( repository, config, diff --git a/borgmatic/actions/compact.py b/borgmatic/actions/compact.py index 970e5d49..b0b070bf 100644 --- a/borgmatic/actions/compact.py +++ b/borgmatic/actions/compact.py @@ -37,9 +37,7 @@ def run_compact( **hook_context, ) if borgmatic.borg.feature.available(borgmatic.borg.feature.Feature.COMPACT, local_borg_version): - logger.info( - f'Compacting segments{dry_run_label}' - ) + logger.info(f'Compacting segments{dry_run_label}') borgmatic.borg.compact.compact_segments( global_arguments.dry_run, repository['path'], @@ -53,9 +51,7 @@ def run_compact( threshold=compact_arguments.threshold, ) else: # pragma: nocover - logger.info( - 'Skipping compact (only available/needed in Borg 1.2+)' - ) + logger.info('Skipping compact (only available/needed in Borg 1.2+)') borgmatic.hooks.command.execute_hook( config.get('after_compact'), diff --git a/borgmatic/actions/create.py b/borgmatic/actions/create.py index 1710351c..51d733d3 100644 --- a/borgmatic/actions/create.py +++ b/borgmatic/actions/create.py @@ -286,9 +286,7 @@ def run_create( logger.info(f'Creating archive{dry_run_label}') working_directory = borgmatic.config.paths.get_working_directory(config) - with borgmatic.config.paths.Runtime_directory( - config - ) as borgmatic_runtime_directory: + with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory: borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_data_source_dumps', config, diff --git a/borgmatic/actions/export_tar.py b/borgmatic/actions/export_tar.py index 7ce6203e..d5c6bacb 100644 --- a/borgmatic/actions/export_tar.py +++ b/borgmatic/actions/export_tar.py @@ -22,9 +22,7 @@ def run_export_tar( if export_tar_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, export_tar_arguments.repository ): - logger.info( - f'Exporting archive {export_tar_arguments.archive} as tar file' - ) + logger.info(f'Exporting archive {export_tar_arguments.archive} as tar file') borgmatic.borg.export_tar.export_tar_archive( global_arguments.dry_run, repository['path'], diff --git a/borgmatic/actions/extract.py b/borgmatic/actions/extract.py index fd20a05b..0216f820 100644 --- a/borgmatic/actions/extract.py +++ b/borgmatic/actions/extract.py @@ -33,9 +33,7 @@ def run_extract( if extract_arguments.repository is None or borgmatic.config.validate.repositories_match( repository, extract_arguments.repository ): - logger.info( - f'Extracting archive {extract_arguments.archive}' - ) + logger.info(f'Extracting archive {extract_arguments.archive}') borgmatic.borg.extract.extract_archive( global_arguments.dry_run, repository['path'], diff --git a/borgmatic/actions/info.py b/borgmatic/actions/info.py index 76bd7554..7dfb6df7 100644 --- a/borgmatic/actions/info.py +++ b/borgmatic/actions/info.py @@ -27,9 +27,7 @@ def run_info( repository, info_arguments.repository ): if not info_arguments.json: - logger.answer( - 'Displaying archive summary information' - ) + logger.answer('Displaying archive summary information') archive_name = borgmatic.borg.repo_list.resolve_archive_name( repository['path'], info_arguments.archive, diff --git a/borgmatic/actions/mount.py b/borgmatic/actions/mount.py index 3bddc619..f7917463 100644 --- a/borgmatic/actions/mount.py +++ b/borgmatic/actions/mount.py @@ -23,9 +23,7 @@ def run_mount( repository, mount_arguments.repository ): if mount_arguments.archive: - logger.info( - f'Mounting archive {mount_arguments.archive}' - ) + logger.info(f'Mounting archive {mount_arguments.archive}') else: # pragma: nocover logger.info('Mounting repository') diff --git a/borgmatic/actions/repo_delete.py b/borgmatic/actions/repo_delete.py index 242ffaa9..29227b15 100644 --- a/borgmatic/actions/repo_delete.py +++ b/borgmatic/actions/repo_delete.py @@ -21,8 +21,7 @@ def run_repo_delete( repository, repo_delete_arguments.repository ): logger.answer( - 'Deleting repository' - + (' cache' if repo_delete_arguments.cache_only else '') + 'Deleting repository' + (' cache' if repo_delete_arguments.cache_only else '') ) borgmatic.borg.repo_delete.delete_repository( diff --git a/borgmatic/actions/repo_info.py b/borgmatic/actions/repo_info.py index 4d4509fa..c8f035c0 100644 --- a/borgmatic/actions/repo_info.py +++ b/borgmatic/actions/repo_info.py @@ -25,9 +25,7 @@ def run_repo_info( repository, repo_info_arguments.repository ): if not repo_info_arguments.json: - logger.answer( - 'Displaying repository summary information' - ) + logger.answer('Displaying repository summary information') json_output = borgmatic.borg.repo_info.display_repository_info( repository['path'], diff --git a/borgmatic/actions/restore.py b/borgmatic/actions/restore.py index 660e7130..595b66ae 100644 --- a/borgmatic/actions/restore.py +++ b/borgmatic/actions/restore.py @@ -172,9 +172,7 @@ def restore_single_dump( Dump(hook_name, data_source['name'], data_source.get('hostname'), data_source.get('port')) ) - logger.info( - f'Restoring data source {dump_metadata}' - ) + logger.info(f'Restoring data source {dump_metadata}') dump_patterns = borgmatic.hooks.dispatch.call_hooks( 'make_data_source_dump_patterns', @@ -443,9 +441,7 @@ def run_restore( logger.info(f'Restoring data sources from archive {restore_arguments.archive}') - with borgmatic.config.paths.Runtime_directory( - config - ) as borgmatic_runtime_directory: + with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory: borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured( 'remove_data_source_dumps', config, diff --git a/borgmatic/actions/transfer.py b/borgmatic/actions/transfer.py index ff94b56e..8a27d16b 100644 --- a/borgmatic/actions/transfer.py +++ b/borgmatic/actions/transfer.py @@ -17,9 +17,7 @@ def run_transfer( ''' Run the "transfer" action for the given repository. ''' - logger.info( - 'Transferring archives to repository' - ) + logger.info('Transferring archives to repository') borgmatic.borg.transfer.transfer_archives( global_arguments.dry_run, repository['path'], diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index fefbd8e9..2ea24086 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -39,7 +39,13 @@ from borgmatic.commands.arguments import parse_arguments from borgmatic.config import checks, collect, validate from borgmatic.hooks import command, dispatch from borgmatic.hooks.monitoring import monitor -from borgmatic.logger import DISABLED, add_custom_log_levels, configure_logging, should_do_markup, set_log_prefix +from borgmatic.logger import ( + DISABLED, + add_custom_log_levels, + configure_logging, + set_log_prefix, + should_do_markup, +) from borgmatic.signals import configure_signals from borgmatic.verbosity import verbosity_to_log_level @@ -131,15 +137,11 @@ def run_configuration(config_filename, config, config_paths, arguments): try: while not repo_queue.empty(): repository, retry_num = repo_queue.get() - set_log_prefix(repository.get('label', repository['path'])) - logger.debug( - 'Running actions for repository' - ) + set_log_prefix(repository.get('label', repository['path'])) + logger.debug('Running actions for repository') timeout = retry_num * retry_wait if timeout: - logger.warning( - f'Sleeping {timeout}s before next retry' - ) + logger.warning(f'Sleeping {timeout}s before next retry') time.sleep(timeout) try: yield from run_actions( @@ -165,9 +167,7 @@ def run_configuration(config_filename, config, config_paths, arguments): log_command_error_output=True, ) ) - logger.warning( - f'Retrying... attempt {retry_num + 1}/{retries}' - ) + logger.warning(f'Retrying... attempt {retry_num + 1}/{retries}') continue if command.considered_soft_failure(error): @@ -819,9 +819,11 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): try: for config_filename, config in configs.items(): - set_log_prefix(config_filename) + set_log_prefix(config_filename) results = list(run_configuration(config_filename, config, config_paths, arguments)) - error_logs = tuple(result for result in results if isinstance(result, logging.LogRecord)) + error_logs = tuple( + result for result in results if isinstance(result, logging.LogRecord) + ) if error_logs: yield from log_error_records('An error occurred') @@ -837,7 +839,7 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): if results: json_results.extend(results) finally: - set_log_prefix(None) + set_log_prefix(None) if 'umount' in arguments: logger.info(f"Unmounting mount point {arguments['umount'].mount_point}") diff --git a/borgmatic/hooks/command.py b/borgmatic/hooks/command.py index d0e55b62..d446ecca 100644 --- a/borgmatic/hooks/command.py +++ b/borgmatic/hooks/command.py @@ -60,9 +60,7 @@ def execute_hook(commands, umask, config_filename, description, dry_run, **conte dry_run_label = ' (dry run; not actually running hooks)' if dry_run else '' context['configuration_filename'] = config_filename - commands = [ - interpolate_context(description, command, context) for command in commands - ] + commands = [interpolate_context(description, command, context) for command in commands] if len(commands) == 1: logger.info(f'Running command for {description} hook{dry_run_label}') diff --git a/borgmatic/hooks/data_source/bootstrap.py b/borgmatic/hooks/data_source/bootstrap.py index b6ac1b4d..67d75421 100644 --- a/borgmatic/hooks/data_source/bootstrap.py +++ b/borgmatic/hooks/data_source/bootstrap.py @@ -83,9 +83,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d for manifest_directory in glob.glob(manifest_glob): manifest_file_path = os.path.join(manifest_directory, 'manifest.json') - logger.debug( - f'Removing bootstrap manifest at {manifest_file_path}{dry_run_label}' - ) + logger.debug(f'Removing bootstrap manifest at {manifest_file_path}{dry_run_label}') if dry_run: continue diff --git a/borgmatic/hooks/data_source/lvm.py b/borgmatic/hooks/data_source/lvm.py index 958ec8f7..8436d0f6 100644 --- a/borgmatic/hooks/data_source/lvm.py +++ b/borgmatic/hooks/data_source/lvm.py @@ -338,9 +338,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d ), 'lvm_snapshots', ) - logger.debug( - f'Looking for snapshots to remove in {snapshots_glob}{dry_run_label}' - ) + logger.debug(f'Looking for snapshots to remove in {snapshots_glob}{dry_run_label}') umount_command = hook_config.get('umount_command', 'umount') for snapshots_directory in glob.glob(snapshots_glob): @@ -364,9 +362,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d if not os.path.isdir(snapshot_mount_path): continue - logger.debug( - f'Unmounting LVM snapshot at {snapshot_mount_path}{dry_run_label}' - ) + logger.debug(f'Unmounting LVM snapshot at {snapshot_mount_path}{dry_run_label}') if dry_run: continue diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index 1d2ac8fd..abf7ba96 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -102,9 +102,7 @@ def execute_dump_command( + ('--result-file', dump_filename) ) - logger.debug( - f'Dumping MariaDB database "{database_name}" to {dump_filename}{dry_run_label}' - ) + logger.debug(f'Dumping MariaDB database "{database_name}" to {dump_filename}{dry_run_label}') if dry_run: return None @@ -155,9 +153,7 @@ def dump_data_sources( for database in databases: dump_path = make_dump_path(borgmatic_runtime_directory) extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None - dump_database_names = database_names_to_dump( - database, extra_environment, dry_run - ) + dump_database_names = database_names_to_dump(database, extra_environment, dry_run) if not dump_database_names: if dry_run: @@ -209,9 +205,7 @@ def remove_data_source_dumps( borgmatic_runtime_directory to construct the destination path. If this is a dry run, then don't actually remove anything. ''' - dump.remove_data_source_dumps( - make_dump_path(borgmatic_runtime_directory), 'MariaDB', dry_run - ) + dump.remove_data_source_dumps(make_dump_path(borgmatic_runtime_directory), 'MariaDB', dry_run) def make_data_source_dump_patterns( diff --git a/borgmatic/hooks/data_source/mongodb.py b/borgmatic/hooks/data_source/mongodb.py index b5c2c744..a1a006a9 100644 --- a/borgmatic/hooks/data_source/mongodb.py +++ b/borgmatic/hooks/data_source/mongodb.py @@ -123,9 +123,7 @@ def remove_data_source_dumps( borgmatic_runtime_directory to construct the destination path. If this is a dry run, then don't actually remove anything. ''' - dump.remove_data_source_dumps( - make_dump_path(borgmatic_runtime_directory), 'MongoDB', dry_run - ) + dump.remove_data_source_dumps(make_dump_path(borgmatic_runtime_directory), 'MongoDB', dry_run) def make_data_source_dump_patterns( diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index 40d9d959..d302d1f5 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -101,9 +101,7 @@ def execute_dump_command( + ('--result-file', dump_filename) ) - logger.debug( - f'Dumping MySQL database "{database_name}" to {dump_filename}{dry_run_label}' - ) + logger.debug(f'Dumping MySQL database "{database_name}" to {dump_filename}{dry_run_label}') if dry_run: return None @@ -154,9 +152,7 @@ def dump_data_sources( for database in databases: dump_path = make_dump_path(borgmatic_runtime_directory) extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None - dump_database_names = database_names_to_dump( - database, extra_environment, dry_run - ) + dump_database_names = database_names_to_dump(database, extra_environment, dry_run) if not dump_database_names: if dry_run: @@ -208,9 +204,7 @@ def remove_data_source_dumps( borgmatic runtime directory to construct the destination path. If this is a dry run, then don't actually remove anything. ''' - dump.remove_data_source_dumps( - make_dump_path(borgmatic_runtime_directory), 'MySQL', dry_run - ) + dump.remove_data_source_dumps(make_dump_path(borgmatic_runtime_directory), 'MySQL', dry_run) def make_data_source_dump_patterns( diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index 95022264..94a6e1f5 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -137,9 +137,7 @@ def dump_data_sources( for database in databases: extra_environment = make_extra_environment(database) dump_path = make_dump_path(borgmatic_runtime_directory) - dump_database_names = database_names_to_dump( - database, extra_environment, dry_run - ) + dump_database_names = database_names_to_dump(database, extra_environment, dry_run) if not dump_database_names: if dry_run: @@ -350,9 +348,7 @@ def restore_data_source_dump( data_source, restore_connection_params=connection_params ) - logger.debug( - f"Restoring PostgreSQL database {data_source['name']}{dry_run_label}" - ) + logger.debug(f"Restoring PostgreSQL database {data_source['name']}{dry_run_label}") if dry_run: return diff --git a/borgmatic/hooks/data_source/sqlite.py b/borgmatic/hooks/data_source/sqlite.py index 15b02bc9..4d5ce9a0 100644 --- a/borgmatic/hooks/data_source/sqlite.py +++ b/borgmatic/hooks/data_source/sqlite.py @@ -105,9 +105,7 @@ def remove_data_source_dumps( borgmatic runtime directory to construct the destination path. If this is a dry run, then don't actually remove anything. ''' - dump.remove_data_source_dumps( - make_dump_path(borgmatic_runtime_directory), 'SQLite', dry_run - ) + dump.remove_data_source_dumps(make_dump_path(borgmatic_runtime_directory), 'SQLite', dry_run) def make_data_source_dump_patterns( diff --git a/borgmatic/hooks/data_source/zfs.py b/borgmatic/hooks/data_source/zfs.py index c1ddb38c..82a71e7a 100644 --- a/borgmatic/hooks/data_source/zfs.py +++ b/borgmatic/hooks/data_source/zfs.py @@ -335,9 +335,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d ), 'zfs_snapshots', ) - logger.debug( - f'Looking for snapshots to remove in {snapshots_glob}{dry_run_label}' - ) + logger.debug(f'Looking for snapshots to remove in {snapshots_glob}{dry_run_label}') umount_command = hook_config.get('umount_command', 'umount') for snapshots_directory in glob.glob(snapshots_glob): @@ -362,9 +360,7 @@ def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, d if not os.path.isdir(snapshot_mount_path): continue - logger.debug( - f'Unmounting ZFS snapshot at {snapshot_mount_path}{dry_run_label}' - ) + logger.debug(f'Unmounting ZFS snapshot at {snapshot_mount_path}{dry_run_label}') if not dry_run: try: diff --git a/borgmatic/logger.py b/borgmatic/logger.py index 888206a2..2a3cb62c 100644 --- a/borgmatic/logger.py +++ b/borgmatic/logger.py @@ -89,7 +89,9 @@ class Multi_stream_handler(logging.Handler): class Console_no_color_formatter(logging.Formatter): def __init__(self, *args, **kwargs): - super(Console_no_color_formatter, self).__init__('{prefix}{message}', style='{', defaults={'prefix': ''}, *args, **kwargs) + super(Console_no_color_formatter, self).__init__( + '{prefix}{message}', style='{', defaults={'prefix': ''}, *args, **kwargs + ) class Color(enum.Enum): @@ -103,7 +105,9 @@ class Color(enum.Enum): class Console_color_formatter(logging.Formatter): def __init__(self, *args, **kwargs): - super(Console_color_formatter, self).__init__('{prefix}{message}', style='{', defaults={'prefix': ''}, *args, **kwargs) + super(Console_color_formatter, self).__init__( + '{prefix}{message}', style='{', defaults={'prefix': ''}, *args, **kwargs + ) def format(self, record): add_custom_log_levels() @@ -255,7 +259,9 @@ def configure_logging( if syslog_path: syslog_handler = logging.handlers.SysLogHandler(address=syslog_path) syslog_handler.setFormatter( - logging.Formatter('borgmatic: {levelname} {prefix}{message}', style='{', defaults={'prefix': ''}) # noqa: FS003 + logging.Formatter( + 'borgmatic: {levelname} {prefix}{message}', style='{', defaults={'prefix': ''} + ) # noqa: FS003 ) syslog_handler.setLevel(syslog_log_level) handlers.append(syslog_handler) @@ -264,7 +270,9 @@ def configure_logging( file_handler = logging.handlers.WatchedFileHandler(log_file) file_handler.setFormatter( logging.Formatter( - log_file_format or '[{asctime}] {levelname}: {prefix}{message}', style='{', defaults={'prefix': ''} # noqa: FS003 + log_file_format or '[{asctime}] {levelname}: {prefix}{message}', + style='{', + defaults={'prefix': ''}, # noqa: FS003 ) ) file_handler.setLevel(log_file_log_level) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 84abcddc..1d103233 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -21,9 +21,7 @@ def test_write_patterns_file_writes_pattern_lines(): def test_write_patterns_file_with_empty_exclude_patterns_does_not_raise(): - module.write_patterns_file( - [], borgmatic_runtime_directory='/run/user/0' - ) + module.write_patterns_file([], borgmatic_runtime_directory='/run/user/0') def test_write_patterns_file_appends_to_existing(): diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py index cc711738..2e4bec78 100644 --- a/tests/unit/hooks/data_source/test_mariadb.py +++ b/tests/unit/hooks/data_source/test_mariadb.py @@ -9,9 +9,7 @@ from borgmatic.hooks.data_source import mariadb as module def test_database_names_to_dump_passes_through_name(): extra_environment = flexmock() - names = module.database_names_to_dump( - {'name': 'foo'}, extra_environment, dry_run=False - ) + names = module.database_names_to_dump({'name': 'foo'}, extra_environment, dry_run=False) assert names == ('foo',) @@ -20,9 +18,7 @@ def test_database_names_to_dump_bails_for_dry_run(): extra_environment = flexmock() flexmock(module).should_receive('execute_command_and_capture_output').never() - names = module.database_names_to_dump( - {'name': 'all'}, extra_environment, dry_run=True - ) + names = module.database_names_to_dump({'name': 'all'}, extra_environment, dry_run=True) assert names == () @@ -34,16 +30,15 @@ def test_database_names_to_dump_queries_mariadb_for_database_names(): extra_environment=extra_environment, ).and_return('foo\nbar\nmysql\n').once() - names = module.database_names_to_dump( - {'name': 'all'}, extra_environment, dry_run=False - ) + names = module.database_names_to_dump({'name': 'all'}, extra_environment, dry_run=False) assert names == ('foo', 'bar') def test_use_streaming_true_for_any_databases(): assert module.use_streaming( - databases=[flexmock(), flexmock()], config=flexmock(), + databases=[flexmock(), flexmock()], + config=flexmock(), ) diff --git a/tests/unit/hooks/data_source/test_mysql.py b/tests/unit/hooks/data_source/test_mysql.py index b9ab1dfb..a056e7e5 100644 --- a/tests/unit/hooks/data_source/test_mysql.py +++ b/tests/unit/hooks/data_source/test_mysql.py @@ -9,9 +9,7 @@ from borgmatic.hooks.data_source import mysql as module def test_database_names_to_dump_passes_through_name(): extra_environment = flexmock() - names = module.database_names_to_dump( - {'name': 'foo'}, extra_environment, dry_run=False - ) + names = module.database_names_to_dump({'name': 'foo'}, extra_environment, dry_run=False) assert names == ('foo',) @@ -20,9 +18,7 @@ def test_database_names_to_dump_bails_for_dry_run(): extra_environment = flexmock() flexmock(module).should_receive('execute_command_and_capture_output').never() - names = module.database_names_to_dump( - {'name': 'all'}, extra_environment, dry_run=True - ) + names = module.database_names_to_dump({'name': 'all'}, extra_environment, dry_run=True) assert names == () @@ -34,16 +30,15 @@ def test_database_names_to_dump_queries_mysql_for_database_names(): extra_environment=extra_environment, ).and_return('foo\nbar\nmysql\n').once() - names = module.database_names_to_dump( - {'name': 'all'}, extra_environment, dry_run=False - ) + names = module.database_names_to_dump({'name': 'all'}, extra_environment, dry_run=False) assert names == ('foo', 'bar') def test_use_streaming_true_for_any_databases(): assert module.use_streaming( - databases=[flexmock(), flexmock()], config=flexmock(), + databases=[flexmock(), flexmock()], + config=flexmock(), ) diff --git a/tests/unit/hooks/data_source/test_sqlite.py b/tests/unit/hooks/data_source/test_sqlite.py index 41526793..02c343a3 100644 --- a/tests/unit/hooks/data_source/test_sqlite.py +++ b/tests/unit/hooks/data_source/test_sqlite.py @@ -7,7 +7,8 @@ from borgmatic.hooks.data_source import sqlite as module def test_use_streaming_true_for_any_databases(): assert module.use_streaming( - databases=[flexmock(), flexmock()], config=flexmock(), + databases=[flexmock(), flexmock()], + config=flexmock(), ) diff --git a/tests/unit/hooks/test_command.py b/tests/unit/hooks/test_command.py index 01d04be0..7f2e7384 100644 --- a/tests/unit/hooks/test_command.py +++ b/tests/unit/hooks/test_command.py @@ -20,9 +20,7 @@ def test_interpolate_context_interpolates_variables(): command = 'ls {foo}{baz} {baz}' # noqa: FS003 context = {'foo': 'bar', 'baz': 'quux'} - assert ( - module.interpolate_context('pre-backup', command, context) == 'ls barquux quux' - ) + assert module.interpolate_context('pre-backup', command, context) == 'ls barquux quux' def test_interpolate_context_escapes_interpolated_variables(): @@ -30,8 +28,7 @@ def test_interpolate_context_escapes_interpolated_variables(): context = {'foo': 'bar', 'inject': 'hi; naughty-command'} assert ( - module.interpolate_context('pre-backup', command, context) - == "ls bar 'hi; naughty-command'" + module.interpolate_context('pre-backup', command, context) == "ls bar 'hi; naughty-command'" ) From b9ec9bb873d35a22a5b47bd67ff0d8e201551f81 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 25 Jan 2025 14:49:39 -0800 Subject: [PATCH 03/12] Don't prefix command output (like Borg output) with the global log prefix (#635). --- borgmatic/borg/list.py | 2 +- borgmatic/borg/repo_list.py | 2 +- borgmatic/execute.py | 21 ++++++++++++++------- borgmatic/logger.py | 14 ++++++++++++++ 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/borgmatic/borg/list.py b/borgmatic/borg/list.py index d995f262..757083f6 100644 --- a/borgmatic/borg/list.py +++ b/borgmatic/borg/list.py @@ -237,7 +237,7 @@ def list_archive( # For each archive listed by Borg, run list on the contents of that archive. for archive in archive_lines: - logger.answer(f'{repository_path}: Listing archive {archive}') + logger.answer(f'Listing archive {archive}') archive_arguments = copy.copy(list_arguments) archive_arguments.archive = archive diff --git a/borgmatic/borg/repo_list.py b/borgmatic/borg/repo_list.py index fbc3b7b9..cfe914d4 100644 --- a/borgmatic/borg/repo_list.py +++ b/borgmatic/borg/repo_list.py @@ -59,7 +59,7 @@ def resolve_archive_name( except IndexError: raise ValueError('No archives found in the repository') - logger.debug(f'{repository_path}: Latest archive is {latest_archive}') + logger.debug(f'Latest archive is {latest_archive}') return latest_archive diff --git a/borgmatic/execute.py b/borgmatic/execute.py index 76442d23..88f55cb3 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -6,6 +6,8 @@ import select import subprocess import textwrap +import borgmatic.logger + logger = logging.getLogger(__name__) @@ -309,13 +311,18 @@ def execute_command( if not run_to_completion: return process - log_outputs( - (process,), - (input_file, output_file), - output_log_level, - borg_local_path, - borg_exit_codes, - ) + try: + original_log_prefix = borgmatic.logger.get_log_prefix() + borgmatic.logger.set_log_prefix(None) # Log command output without any prefix. + log_outputs( + (process,), + (input_file, output_file), + output_log_level, + borg_local_path, + borg_exit_codes, + ) + finally: + borgmatic.logger.set_log_prefix(original_log_prefix) def execute_command_and_capture_output( diff --git a/borgmatic/logger.py b/borgmatic/logger.py index 2a3cb62c..5ef16259 100644 --- a/borgmatic/logger.py +++ b/borgmatic/logger.py @@ -186,6 +186,20 @@ def add_custom_log_levels(): # pragma: no cover add_logging_level('DISABLED', DISABLED) +def get_log_prefix(): + ''' + Return the log prefix (as set with set_log_prefix()) for the first log handler. If there is no + such prefix, return None. + ''' + for handler in logging.getLogger().handlers: + defaults = handler.formatter._style._defaults + + if not defaults: + return None + + return defaults.get('prefix').rstrip().rstrip(':') + + def set_log_prefix(prefix): ''' Given a prefix string, set it onto the formatter defaults for every logging handler so that it From e0059de711aa0d2a68da545483f53f26febcce72 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 25 Jan 2025 21:56:41 -0800 Subject: [PATCH 04/12] Add log prefix context manager to make prefix cleanup/restoration easier (#635). --- borgmatic/commands/borgmatic.py | 19 ++++------- borgmatic/execute.py | 6 +--- borgmatic/logger.py | 58 +++++++++++++++++++++++---------- 3 files changed, 49 insertions(+), 34 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 2ea24086..1e46205b 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -41,9 +41,9 @@ from borgmatic.hooks import command, dispatch from borgmatic.hooks.monitoring import monitor from borgmatic.logger import ( DISABLED, + Log_prefix, add_custom_log_levels, configure_logging, - set_log_prefix, should_do_markup, ) from borgmatic.signals import configure_signals @@ -134,10 +134,10 @@ def run_configuration(config_filename, config, config_paths, arguments): (repo, 0), ) - try: - while not repo_queue.empty(): - repository, retry_num = repo_queue.get() - set_log_prefix(repository.get('label', repository['path'])) + while not repo_queue.empty(): + repository, retry_num = repo_queue.get() + + with Log_prefix(repository.get('label', repository['path'])): logger.debug('Running actions for repository') timeout = retry_num * retry_wait if timeout: @@ -179,8 +179,6 @@ def run_configuration(config_filename, config, config_paths, arguments): ) encountered_error = error error_repository = repository['path'] - finally: - set_log_prefix(config_filename) try: if monitoring_hooks_are_activated: @@ -817,9 +815,8 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): # Execute the actions corresponding to each configuration file. json_results = [] - try: - for config_filename, config in configs.items(): - set_log_prefix(config_filename) + for config_filename, config in configs.items(): + with Log_prefix(config_filename): results = list(run_configuration(config_filename, config, config_paths, arguments)) error_logs = tuple( result for result in results if isinstance(result, logging.LogRecord) @@ -838,8 +835,6 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments): ) if results: json_results.extend(results) - finally: - set_log_prefix(None) if 'umount' in arguments: logger.info(f"Unmounting mount point {arguments['umount'].mount_point}") diff --git a/borgmatic/execute.py b/borgmatic/execute.py index 88f55cb3..67a6aac3 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -311,9 +311,7 @@ def execute_command( if not run_to_completion: return process - try: - original_log_prefix = borgmatic.logger.get_log_prefix() - borgmatic.logger.set_log_prefix(None) # Log command output without any prefix. + with borgmatic.logger.Log_prefix(None): # Log command output without any prefix. log_outputs( (process,), (input_file, output_file), @@ -321,8 +319,6 @@ def execute_command( borg_local_path, borg_exit_codes, ) - finally: - borgmatic.logger.set_log_prefix(original_log_prefix) def execute_command_and_capture_output( diff --git a/borgmatic/logger.py b/borgmatic/logger.py index 5ef16259..8869168d 100644 --- a/borgmatic/logger.py +++ b/borgmatic/logger.py @@ -186,28 +186,52 @@ def add_custom_log_levels(): # pragma: no cover add_logging_level('DISABLED', DISABLED) -def get_log_prefix(): +class Log_prefix: ''' - Return the log prefix (as set with set_log_prefix()) for the first log handler. If there is no - such prefix, return None. + A Python context manager for setting a log prefix so that it shows up in every subsequent + logging message for the duration of the context manager. For this to work, it relies on each + logging formatter to be initialized with "{prefix}" somewhere in its logging format. + + Example use as a context manager: + + + with borgmatic.logger.Log_prefix('myprefix'): + do_something_that_logs() + + For the scope of that "with" statement, any logs created are prefixed with "myprefix: ". + Afterwards, the prefix gets restored to whatever it was prior to the context manager. ''' - for handler in logging.getLogger().handlers: - defaults = handler.formatter._style._defaults + def __init__(self, prefix): + ''' + Given the desired log prefix, save it for use below. Set prefix to None to disable any + prefix from getting logged. + ''' + self.prefix = prefix + self.original_prefix = None - if not defaults: - return None + def __enter__(self): + ''' + Set the prefix onto the formatter defaults for every logging handler so that the prefix ends + up in every log message. But first, save off any original prefix so that it can be restored + below. + ''' + try: + self.original_prefix = next( + handler.formatter._style._defaults.get('prefix').rstrip().rstrip(':') + for handler in logging.getLogger().handlers + ) + except (StopIteration, AttributeError): + self.original_prefix = None - return defaults.get('prefix').rstrip().rstrip(':') + for handler in logging.getLogger().handlers: + handler.formatter._style._defaults = {'prefix': f'{self.prefix}: ' if self.prefix else ''} - -def set_log_prefix(prefix): - ''' - Given a prefix string, set it onto the formatter defaults for every logging handler so that it - shows up in every subsequent logging message. For this to work, this relies on each logging - formatter to be initialized with "{prefix}" somewhere in its logging format. - ''' - for handler in logging.getLogger().handlers: - handler.formatter._style._defaults = {'prefix': f'{prefix}: ' if prefix else ''} + def __exit__(self, exception, value, traceback): + ''' + Restore any original prefix. + ''' + for handler in logging.getLogger().handlers: + handler.formatter._style._defaults = {'prefix': f'{self.original_prefix}: ' if self.original_prefix else ''} def configure_logging( From 803361b850f87fcc2a89c0c01de0d49db64b3f64 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 26 Jan 2025 09:12:18 -0800 Subject: [PATCH 05/12] Some text fixes (#635). --- borgmatic/logger.py | 10 ++++++---- tests/unit/test_logger.py | 7 +++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/borgmatic/logger.py b/borgmatic/logger.py index 8869168d..a2eceacd 100644 --- a/borgmatic/logger.py +++ b/borgmatic/logger.py @@ -298,8 +298,10 @@ def configure_logging( syslog_handler = logging.handlers.SysLogHandler(address=syslog_path) syslog_handler.setFormatter( logging.Formatter( - 'borgmatic: {levelname} {prefix}{message}', style='{', defaults={'prefix': ''} - ) # noqa: FS003 + 'borgmatic: {levelname} {prefix}{message}', # noqa: FS003 + style='{', + defaults={'prefix': ''}, + ) ) syslog_handler.setLevel(syslog_log_level) handlers.append(syslog_handler) @@ -308,9 +310,9 @@ def configure_logging( file_handler = logging.handlers.WatchedFileHandler(log_file) file_handler.setFormatter( logging.Formatter( - log_file_format or '[{asctime}] {levelname}: {prefix}{message}', + log_file_format or '[{asctime}] {levelname}: {prefix}{message}', # noqa: FS003 style='{', - defaults={'prefix': ''}, # noqa: FS003 + defaults={'prefix': ''}, ) ) file_handler.setLevel(log_file_log_level) diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index 442d65c0..cc6619ed 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -177,8 +177,9 @@ def test_multi_stream_handler_logs_to_handler_for_log_level(): def test_console_color_formatter_format_includes_log_message(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.ANSWER + flexmock(module.logging.Formatter).should_receive('format') plain_message = 'uh oh' - record = flexmock(levelno=logging.CRITICAL, msg=plain_message) + record = flexmock(levelno=logging.CRITICAL) colored_message = module.Console_color_formatter().format(record) @@ -409,7 +410,9 @@ def test_configure_logging_to_log_file_formats_with_custom_log_format(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.ANSWER flexmock(module.logging).should_receive('Formatter').with_args( - '{message}', style='{' # noqa: FS003 + '{message}', # noqa: FS003 + style='{', + defaults={'prefix': ''}, ).once() fake_formatter = flexmock() flexmock(module).should_receive('Console_color_formatter').and_return(fake_formatter) From 7d56641f562c4d61e0d71e8fdc24d04a88ce0732 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 26 Jan 2025 12:13:29 -0800 Subject: [PATCH 06/12] Get existing unit tests passing (#635). --- borgmatic/borg/change_passphrase.py | 2 +- borgmatic/borg/compact.py | 2 +- borgmatic/borg/create.py | 6 +- borgmatic/borg/export_key.py | 2 +- borgmatic/borg/export_tar.py | 2 +- borgmatic/borg/repo_create.py | 4 +- borgmatic/commands/borgmatic.py | 4 +- tests/unit/actions/test_restore.py | 8 +-- tests/unit/borg/test_create.py | 6 +- tests/unit/commands/test_borgmatic.py | 66 ++++++++++++++----- tests/unit/config/test_paths.py | 30 ++++----- tests/unit/hooks/data_source/test_dump.py | 6 +- tests/unit/hooks/data_source/test_mariadb.py | 18 +---- tests/unit/hooks/data_source/test_mongodb.py | 17 ----- tests/unit/hooks/data_source/test_mysql.py | 18 +---- .../unit/hooks/data_source/test_postgresql.py | 44 +++---------- tests/unit/hooks/data_source/test_sqlite.py | 10 --- tests/unit/hooks/test_dispatch.py | 32 ++++----- tests/unit/test_execute.py | 8 +++ tests/unit/test_logger.py | 2 +- 20 files changed, 122 insertions(+), 165 deletions(-) diff --git a/borgmatic/borg/change_passphrase.py b/borgmatic/borg/change_passphrase.py index 70c0065f..79f9781d 100644 --- a/borgmatic/borg/change_passphrase.py +++ b/borgmatic/borg/change_passphrase.py @@ -41,7 +41,7 @@ def change_passphrase( ) if global_arguments.dry_run: - logger.info(f'{repository_path}: Skipping change password (dry run)') + logger.info('Skipping change password (dry run)') return # If the original passphrase is set programmatically, then Borg won't prompt for a new one! So diff --git a/borgmatic/borg/compact.py b/borgmatic/borg/compact.py index c7fef483..eb16c6fa 100644 --- a/borgmatic/borg/compact.py +++ b/borgmatic/borg/compact.py @@ -43,7 +43,7 @@ def compact_segments( ) if dry_run: - logging.info(f'{repository_path}: Skipping compact (dry run)') + logging.info('Skipping compact (dry run)') return execute_command( diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 1fb5284d..27ba111f 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -295,12 +295,12 @@ def make_base_create_command( # cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True. if stream_processes and not config.get('read_special'): logger.warning( - f'{repository_path}: Ignoring configured "read_special" value of false, as true is needed for database hooks.' + 'Ignoring configured "read_special" value of false, as true is needed for database hooks.' ) borg_environment = environment.make_environment(config) working_directory = borgmatic.config.paths.get_working_directory(config) - logger.debug(f'{repository_path}: Collecting special file paths') + logger.debug('Collecting special file paths') special_file_paths = collect_special_file_paths( dry_run, create_flags + create_positional_arguments, @@ -318,7 +318,7 @@ def make_base_create_command( placeholder=' ...', ) logger.warning( - f'{repository_path}: Excluding special files to prevent Borg from hanging: {truncated_special_file_paths}' + f'Excluding special files to prevent Borg from hanging: {truncated_special_file_paths}' ) patterns_file = write_patterns_file( tuple( diff --git a/borgmatic/borg/export_key.py b/borgmatic/borg/export_key.py index cd0e8cf4..768b52cc 100644 --- a/borgmatic/borg/export_key.py +++ b/borgmatic/borg/export_key.py @@ -60,7 +60,7 @@ def export_key( ) if global_arguments.dry_run: - logger.info(f'{repository_path}: Skipping key export (dry run)') + logger.info('Skipping key export (dry run)') return execute_command( diff --git a/borgmatic/borg/export_tar.py b/borgmatic/borg/export_tar.py index 3adc8ae6..728755d5 100644 --- a/borgmatic/borg/export_tar.py +++ b/borgmatic/borg/export_tar.py @@ -63,7 +63,7 @@ def export_tar_archive( output_log_level = logging.INFO if dry_run: - logging.info(f'{repository_path}: Skipping export to tar file (dry run)') + logging.info('Skipping export to tar file (dry run)') return execute_command( diff --git a/borgmatic/borg/repo_create.py b/borgmatic/borg/repo_create.py index 068ad373..7ccf85ba 100644 --- a/borgmatic/borg/repo_create.py +++ b/borgmatic/borg/repo_create.py @@ -57,7 +57,7 @@ def create_repository( f'Requested encryption mode "{encryption_mode}" does not match existing repository encryption mode "{repository_encryption_mode}"' ) - logger.info(f'{repository_path}: Repository already exists. Skipping creation.') + logger.info('Repository already exists. Skipping creation.') return except subprocess.CalledProcessError as error: if error.returncode not in REPO_INFO_REPOSITORY_NOT_FOUND_EXIT_CODES: @@ -91,7 +91,7 @@ def create_repository( ) if dry_run: - logging.info(f'{repository_path}: Skipping repository creation (dry run)') + logging.info('Skipping repository creation (dry run)') return # Do not capture output here, so as to support interactive prompts. diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 1e46205b..112a7691 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -161,7 +161,7 @@ def run_configuration(config_filename, config, config_paths, arguments): ) tuple( # Consume the generator so as to trigger logging. log_error_records( - f'{repository.get("label", repository["path"])}: Error running actions for repository', + 'Error running actions for repository', error, levelno=logging.WARNING, log_command_error_output=True, @@ -194,7 +194,7 @@ def run_configuration(config_filename, config, config_paths, arguments): except (OSError, CalledProcessError) as error: if not command.considered_soft_failure(error): encountered_error = error - yield from log_error_records(f'{repository["path"]}: Error pinging monitor', error) + yield from log_error_records('Error pinging monitor', error) if not encountered_error: try: diff --git a/tests/unit/actions/test_restore.py b/tests/unit/actions/test_restore.py index 6fedb367..d930d670 100644 --- a/tests/unit/actions/test_restore.py +++ b/tests/unit/actions/test_restore.py @@ -273,7 +273,7 @@ def test_strip_path_prefix_from_extracted_dump_destination_renames_first_matchin def test_restore_single_dump_extracts_and_restores_single_file_dump(): flexmock(module).should_receive('render_dump_metadata').and_return('test') flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').with_args( - 'make_data_source_dump_patterns', object, object, object, object, object + 'make_data_source_dump_patterns', object, object, object, object ).and_return({'postgresql': flexmock()}) flexmock(module.tempfile).should_receive('mkdtemp').never() flexmock(module.borgmatic.hooks.data_source.dump).should_receive( @@ -313,7 +313,7 @@ def test_restore_single_dump_extracts_and_restores_single_file_dump(): def test_restore_single_dump_extracts_and_restores_directory_dump(): flexmock(module).should_receive('render_dump_metadata').and_return('test') flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').with_args( - 'make_data_source_dump_patterns', object, object, object, object, object + 'make_data_source_dump_patterns', object, object, object, object ).and_return({'postgresql': flexmock()}) flexmock(module.tempfile).should_receive('mkdtemp').once().and_return( '/run/user/0/borgmatic/tmp1234' @@ -355,7 +355,7 @@ def test_restore_single_dump_extracts_and_restores_directory_dump(): def test_restore_single_dump_with_directory_dump_error_cleans_up_temporary_directory(): flexmock(module).should_receive('render_dump_metadata').and_return('test') flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').with_args( - 'make_data_source_dump_patterns', object, object, object, object, object + 'make_data_source_dump_patterns', object, object, object, object ).and_return({'postgresql': flexmock()}) flexmock(module.tempfile).should_receive('mkdtemp').once().and_return( '/run/user/0/borgmatic/tmp1234' @@ -398,7 +398,7 @@ def test_restore_single_dump_with_directory_dump_error_cleans_up_temporary_direc def test_restore_single_dump_with_directory_dump_and_dry_run_skips_directory_move_and_cleanup(): flexmock(module).should_receive('render_dump_metadata').and_return('test') flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hooks').with_args( - 'make_data_source_dump_patterns', object, object, object, object, object + 'make_data_source_dump_patterns', object, object, object, object ).and_return({'postgresql': flexmock()}) flexmock(module.tempfile).should_receive('mkdtemp').once().and_return('/run/borgmatic/tmp1234') flexmock(module.borgmatic.hooks.data_source.dump).should_receive( diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 1d103233..a13c4184 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -643,7 +643,7 @@ def test_make_base_create_command_with_stream_processes_ignores_read_special_fal patterns_file = flexmock(name='patterns') flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('write_patterns_file').with_args( - patterns, '/run/borgmatic', object + patterns, '/run/borgmatic' ).and_return(patterns_file) flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.flags).should_receive('get_default_archive_name_format').and_return( @@ -666,7 +666,6 @@ def test_make_base_create_command_with_stream_processes_ignores_read_special_fal ), ), '/run/borgmatic', - 'repo', patterns_file=patterns_file, ).and_return(patterns_file).once() flexmock(module).should_receive('make_exclude_flags').and_return(()) @@ -694,7 +693,7 @@ def test_make_base_create_command_with_stream_processes_ignores_read_special_fal def test_make_base_create_command_without_patterns_and_with_stream_processes_ignores_read_special_false_and_excludes_special_files(): flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None) flexmock(module).should_receive('write_patterns_file').with_args( - [], '/run/borgmatic', object + [], '/run/borgmatic' ).and_return(None) flexmock(module).should_receive('make_list_filter_flags').and_return('FOO') flexmock(module.flags).should_receive('get_default_archive_name_format').and_return( @@ -717,7 +716,6 @@ def test_make_base_create_command_without_patterns_and_with_stream_processes_ign ), ), '/run/borgmatic', - 'repo', patterns_file=None, ).and_return(flexmock(name='patterns')).once() flexmock(module).should_receive('make_exclude_flags').and_return(()) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index bba36928..e5e599e1 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -32,6 +32,7 @@ def test_run_configuration_runs_actions_for_each_repository(): flexmock(module).should_receive('get_skip_actions').and_return([]) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) expected_results = [flexmock(), flexmock()] + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_return(expected_results[:1]).and_return( expected_results[1:] ) @@ -47,6 +48,7 @@ def test_run_configuration_with_skip_actions_does_not_raise(): flexmock(module).should_receive('verbosity_to_log_level').and_return(logging.INFO) flexmock(module).should_receive('get_skip_actions').and_return(['compact']) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_return(flexmock()).and_return(flexmock()) config = {'repositories': [{'path': 'foo'}, {'path': 'bar'}], 'skip_actions': ['compact']} arguments = {'global': flexmock(monitoring_verbosity=1)} @@ -60,6 +62,7 @@ def test_run_configuration_with_invalid_borg_version_errors(): flexmock(module.borg_version).should_receive('local_borg_version').and_raise(ValueError) flexmock(module.command).should_receive('execute_hook').never() flexmock(module.dispatch).should_receive('call_hooks').never() + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').never() config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'prune': flexmock()} @@ -76,6 +79,7 @@ def test_run_configuration_logs_monitor_start_error(): ).and_return(None).and_return(None) expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').never() config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} @@ -92,6 +96,7 @@ def test_run_configuration_bails_for_monitor_start_soft_failure(): error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') flexmock(module.dispatch).should_receive('call_hooks').and_raise(error).and_return(None) flexmock(module).should_receive('log_error_records').never() + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').never() config = {'repositories': [{'path': 'foo'}, {'path': 'bar'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} @@ -109,6 +114,7 @@ def test_run_configuration_logs_actions_error(): flexmock(module.dispatch).should_receive('call_hooks') expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError) config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False)} @@ -125,6 +131,7 @@ def test_run_configuration_skips_remaining_actions_for_actions_soft_failure_but_ flexmock(module.dispatch).should_receive('call_hooks').times(5) error = subprocess.CalledProcessError(borgmatic.hooks.command.SOFT_FAIL_EXIT_CODE, 'try again') log = flexmock() + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').twice().and_raise(error).and_yield(log) flexmock(module).should_receive('log_error_records').never() flexmock(module.command).should_receive('considered_soft_failure').and_return(True) @@ -145,6 +152,7 @@ def test_run_configuration_logs_monitor_log_error(): ).and_raise(OSError) expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_return([]) config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} @@ -163,6 +171,7 @@ def test_run_configuration_still_pings_monitor_for_monitor_log_soft_failure(): None ).and_raise(error).and_return(None).and_return(None).times(5) flexmock(module).should_receive('log_error_records').never() + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_return([]) flexmock(module.command).should_receive('considered_soft_failure').and_return(True) config = {'repositories': [{'path': 'foo'}]} @@ -182,6 +191,7 @@ def test_run_configuration_logs_monitor_finish_error(): ).and_return(None).and_raise(OSError) expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_return([]) config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} @@ -200,6 +210,7 @@ def test_run_configuration_bails_for_monitor_finish_soft_failure(): None ).and_raise(None).and_raise(error) flexmock(module).should_receive('log_error_records').never() + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_return([]) flexmock(module.command).should_receive('considered_soft_failure').and_return(True) config = {'repositories': [{'path': 'foo'}]} @@ -216,6 +227,7 @@ def test_run_configuration_does_not_call_monitoring_hooks_if_monitoring_hooks_ar flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.dispatch).should_receive('call_hooks').never() + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_return([]) config = {'repositories': [{'path': 'foo'}]} @@ -233,6 +245,7 @@ def test_run_configuration_logs_on_error_hook_error(): flexmock(module).should_receive('log_error_records').and_return( expected_results[:1] ).and_return(expected_results[1:]) + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError) config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} @@ -250,6 +263,7 @@ def test_run_configuration_bails_for_on_error_hook_soft_failure(): flexmock(module.command).should_receive('execute_hook').and_raise(error) expected_results = [flexmock()] flexmock(module).should_receive('log_error_records').and_return(expected_results) + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError) config = {'repositories': [{'path': 'foo'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} @@ -265,6 +279,7 @@ def test_run_configuration_retries_soft_error(): flexmock(module).should_receive('get_skip_actions').and_return([]) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.command).should_receive('execute_hook') + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError).and_return([]) flexmock(module).should_receive('log_error_records').and_return([flexmock()]).once() config = {'repositories': [{'path': 'foo'}], 'retries': 1} @@ -279,16 +294,17 @@ def test_run_configuration_retries_hard_error(): flexmock(module).should_receive('get_skip_actions').and_return([]) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.command).should_receive('execute_hook') + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError).times(2) flexmock(module).should_receive('log_error_records').with_args( - 'foo: Error running actions for repository', + 'Error running actions for repository', OSError, levelno=logging.WARNING, log_command_error_output=True, ).and_return([flexmock()]) error_logs = [flexmock()] flexmock(module).should_receive('log_error_records').with_args( - 'foo: Error running actions for repository', + 'Error running actions for repository', OSError, ).and_return(error_logs) config = {'repositories': [{'path': 'foo'}], 'retries': 1} @@ -302,13 +318,14 @@ def test_run_configuration_repos_ordered(): flexmock(module).should_receive('get_skip_actions').and_return([]) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.command).should_receive('execute_hook') + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError).times(2) expected_results = [flexmock(), flexmock()] flexmock(module).should_receive('log_error_records').with_args( - 'foo: Error running actions for repository', OSError + 'Error running actions for repository', OSError ).and_return(expected_results[:1]).ordered() flexmock(module).should_receive('log_error_records').with_args( - 'bar: Error running actions for repository', OSError + 'Error running actions for repository', OSError ).and_return(expected_results[1:]).ordered() config = {'repositories': [{'path': 'foo'}, {'path': 'bar'}]} arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()} @@ -321,26 +338,27 @@ def test_run_configuration_retries_round_robin(): flexmock(module).should_receive('get_skip_actions').and_return([]) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.command).should_receive('execute_hook') + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) flexmock(module).should_receive('log_error_records').with_args( - 'foo: Error running actions for repository', + 'Error running actions for repository', OSError, levelno=logging.WARNING, log_command_error_output=True, ).and_return([flexmock()]).ordered() flexmock(module).should_receive('log_error_records').with_args( - 'bar: Error running actions for repository', + 'Error running actions for repository', OSError, levelno=logging.WARNING, log_command_error_output=True, ).and_return([flexmock()]).ordered() foo_error_logs = [flexmock()] flexmock(module).should_receive('log_error_records').with_args( - 'foo: Error running actions for repository', OSError + 'Error running actions for repository', OSError ).and_return(foo_error_logs).ordered() bar_error_logs = [flexmock()] flexmock(module).should_receive('log_error_records').with_args( - 'bar: Error running actions for repository', OSError + 'Error running actions for repository', OSError ).and_return(bar_error_logs).ordered() config = { 'repositories': [{'path': 'foo'}, {'path': 'bar'}], @@ -356,24 +374,25 @@ def test_run_configuration_retries_one_passes(): flexmock(module).should_receive('get_skip_actions').and_return([]) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.command).should_receive('execute_hook') + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return( [] ).and_raise(OSError).times(4) flexmock(module).should_receive('log_error_records').with_args( - 'foo: Error running actions for repository', + 'Error running actions for repository', OSError, levelno=logging.WARNING, log_command_error_output=True, ).and_return([flexmock()]).ordered() flexmock(module).should_receive('log_error_records').with_args( - 'bar: Error running actions for repository', + 'Error running actions for repository', OSError, levelno=logging.WARNING, log_command_error_output=True, ).and_return(flexmock()).ordered() error_logs = [flexmock()] flexmock(module).should_receive('log_error_records').with_args( - 'bar: Error running actions for repository', OSError + 'Error running actions for repository', OSError ).and_return(error_logs).ordered() config = { 'repositories': [{'path': 'foo'}, {'path': 'bar'}], @@ -389,9 +408,10 @@ def test_run_configuration_retry_wait(): flexmock(module).should_receive('get_skip_actions').and_return([]) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.command).should_receive('execute_hook') + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError).times(4) flexmock(module).should_receive('log_error_records').with_args( - 'foo: Error running actions for repository', + 'Error running actions for repository', OSError, levelno=logging.WARNING, log_command_error_output=True, @@ -399,7 +419,7 @@ def test_run_configuration_retry_wait(): flexmock(time).should_receive('sleep').with_args(10).and_return().ordered() flexmock(module).should_receive('log_error_records').with_args( - 'foo: Error running actions for repository', + 'Error running actions for repository', OSError, levelno=logging.WARNING, log_command_error_output=True, @@ -407,7 +427,7 @@ def test_run_configuration_retry_wait(): flexmock(time).should_receive('sleep').with_args(20).and_return().ordered() flexmock(module).should_receive('log_error_records').with_args( - 'foo: Error running actions for repository', + 'Error running actions for repository', OSError, levelno=logging.WARNING, log_command_error_output=True, @@ -416,7 +436,7 @@ def test_run_configuration_retry_wait(): flexmock(time).should_receive('sleep').with_args(30).and_return().ordered() error_logs = [flexmock()] flexmock(module).should_receive('log_error_records').with_args( - 'foo: Error running actions for repository', OSError + 'Error running actions for repository', OSError ).and_return(error_logs).ordered() config = { 'repositories': [{'path': 'foo'}], @@ -433,17 +453,18 @@ def test_run_configuration_retries_timeout_multiple_repos(): flexmock(module).should_receive('get_skip_actions').and_return([]) flexmock(module.borg_version).should_receive('local_borg_version').and_return(flexmock()) flexmock(module.command).should_receive('execute_hook') + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return( [] ).and_raise(OSError).times(4) flexmock(module).should_receive('log_error_records').with_args( - 'foo: Error running actions for repository', + 'Error running actions for repository', OSError, levelno=logging.WARNING, log_command_error_output=True, ).and_return([flexmock()]).ordered() flexmock(module).should_receive('log_error_records').with_args( - 'bar: Error running actions for repository', + 'Error running actions for repository', OSError, levelno=logging.WARNING, log_command_error_output=True, @@ -456,7 +477,7 @@ def test_run_configuration_retries_timeout_multiple_repos(): flexmock(time).should_receive('sleep').with_args(10).and_return().ordered() error_logs = [flexmock()] flexmock(module).should_receive('log_error_records').with_args( - 'bar: Error running actions for repository', OSError + 'Error running actions for repository', OSError ).and_return(error_logs).ordered() config = { 'repositories': [{'path': 'foo'}, {'path': 'bar'}], @@ -1370,6 +1391,7 @@ def test_collect_highlander_action_summary_logs_error_on_run_validate_failure(): def test_collect_configuration_run_summary_logs_info_for_success(): flexmock(module.command).should_receive('execute_hook').never() flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return([]) arguments = {} @@ -1384,6 +1406,7 @@ def test_collect_configuration_run_summary_logs_info_for_success(): def test_collect_configuration_run_summary_executes_hooks_for_create(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return([]) arguments = {'create': flexmock(), 'global': flexmock(monitoring_verbosity=1, dry_run=False)} @@ -1398,6 +1421,7 @@ def test_collect_configuration_run_summary_executes_hooks_for_create(): def test_collect_configuration_run_summary_logs_info_for_success_with_extract(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return([]) arguments = {'extract': flexmock(repository='repo')} @@ -1429,6 +1453,7 @@ def test_collect_configuration_run_summary_logs_extract_with_repository_error(): def test_collect_configuration_run_summary_logs_info_for_success_with_mount(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return([]) arguments = {'mount': flexmock(repository='repo')} @@ -1488,6 +1513,7 @@ def test_collect_configuration_run_summary_logs_pre_hook_error(): def test_collect_configuration_run_summary_logs_post_hook_error(): flexmock(module.command).should_receive('execute_hook').and_return(None).and_raise(ValueError) flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return([]) expected_logs = (flexmock(),) flexmock(module).should_receive('log_error_records').and_return(expected_logs) @@ -1521,6 +1547,7 @@ def test_collect_configuration_run_summary_logs_for_list_with_archive_and_reposi def test_collect_configuration_run_summary_logs_info_for_success_with_list(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return([]) arguments = {'list': flexmock(repository='repo', archive=None)} @@ -1535,6 +1562,7 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_list(): def test_collect_configuration_run_summary_logs_run_configuration_error(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return( [logging.makeLogRecord(dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg='Error'))] ) @@ -1552,6 +1580,7 @@ def test_collect_configuration_run_summary_logs_run_configuration_error(): def test_collect_configuration_run_summary_logs_run_umount_error(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return([]) flexmock(module.borg_umount).should_receive('unmount_archive').and_raise(OSError) flexmock(module).should_receive('log_error_records').and_return( @@ -1570,6 +1599,7 @@ def test_collect_configuration_run_summary_logs_run_umount_error(): def test_collect_configuration_run_summary_logs_outputs_merged_json_results(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') + flexmock(module).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('run_configuration').and_return(['foo', 'bar']).and_return( ['baz'] ) diff --git a/tests/unit/config/test_paths.py b/tests/unit/config/test_paths.py index a63c86e1..12975b9d 100644 --- a/tests/unit/config/test_paths.py +++ b/tests/unit/config/test_paths.py @@ -66,7 +66,7 @@ def test_runtime_directory_uses_config_option(): flexmock(module.os).should_receive('makedirs') config = {'user_runtime_directory': '/run', 'borgmatic_source_directory': '/nope'} - with module.Runtime_directory(config, 'prefix') as borgmatic_runtime_directory: + with module.Runtime_directory(config) as borgmatic_runtime_directory: assert borgmatic_runtime_directory == '/run/./borgmatic' @@ -75,7 +75,7 @@ def test_runtime_directory_uses_config_option_without_adding_duplicate_borgmatic flexmock(module.os).should_receive('makedirs') config = {'user_runtime_directory': '/run/borgmatic', 'borgmatic_source_directory': '/nope'} - with module.Runtime_directory(config, 'prefix') as borgmatic_runtime_directory: + with module.Runtime_directory(config) as borgmatic_runtime_directory: assert borgmatic_runtime_directory == '/run/./borgmatic' @@ -85,7 +85,7 @@ def test_runtime_directory_with_relative_config_option_errors(): with pytest.raises(ValueError): with module.Runtime_directory( - config, 'prefix' + config ) as borgmatic_runtime_directory: # noqa: F841 pass @@ -97,7 +97,7 @@ def test_runtime_directory_falls_back_to_xdg_runtime_dir(): ) flexmock(module.os).should_receive('makedirs') - with module.Runtime_directory({}, 'prefix') as borgmatic_runtime_directory: + with module.Runtime_directory({}) as borgmatic_runtime_directory: assert borgmatic_runtime_directory == '/run/./borgmatic' @@ -108,7 +108,7 @@ def test_runtime_directory_falls_back_to_xdg_runtime_dir_without_adding_duplicat ) flexmock(module.os).should_receive('makedirs') - with module.Runtime_directory({}, 'prefix') as borgmatic_runtime_directory: + with module.Runtime_directory({}) as borgmatic_runtime_directory: assert borgmatic_runtime_directory == '/run/./borgmatic' @@ -117,7 +117,7 @@ def test_runtime_directory_with_relative_xdg_runtime_dir_errors(): flexmock(module.os).should_receive('makedirs').never() with pytest.raises(ValueError): - with module.Runtime_directory({}, 'prefix') as borgmatic_runtime_directory: # noqa: F841 + with module.Runtime_directory({}) as borgmatic_runtime_directory: # noqa: F841 pass @@ -129,7 +129,7 @@ def test_runtime_directory_falls_back_to_runtime_directory(): ) flexmock(module.os).should_receive('makedirs') - with module.Runtime_directory({}, 'prefix') as borgmatic_runtime_directory: + with module.Runtime_directory({}) as borgmatic_runtime_directory: assert borgmatic_runtime_directory == '/run/./borgmatic' @@ -141,7 +141,7 @@ def test_runtime_directory_falls_back_to_runtime_directory_without_adding_duplic ) flexmock(module.os).should_receive('makedirs') - with module.Runtime_directory({}, 'prefix') as borgmatic_runtime_directory: + with module.Runtime_directory({}) as borgmatic_runtime_directory: assert borgmatic_runtime_directory == '/run/./borgmatic' @@ -153,7 +153,7 @@ def test_runtime_directory_with_relative_runtime_directory_errors(): flexmock(module.os).should_receive('makedirs').never() with pytest.raises(ValueError): - with module.Runtime_directory({}, 'prefix') as borgmatic_runtime_directory: # noqa: F841 + with module.Runtime_directory({}) as borgmatic_runtime_directory: # noqa: F841 pass @@ -171,7 +171,7 @@ def test_runtime_directory_falls_back_to_tmpdir_and_adds_temporary_subdirectory_ ).and_return(temporary_directory) flexmock(module.os).should_receive('makedirs') - with module.Runtime_directory({}, 'prefix') as borgmatic_runtime_directory: + with module.Runtime_directory({}) as borgmatic_runtime_directory: assert borgmatic_runtime_directory == '/run/borgmatic-1234/./borgmatic' @@ -185,7 +185,7 @@ def test_runtime_directory_with_relative_tmpdir_errors(): flexmock(module.os).should_receive('makedirs').never() with pytest.raises(ValueError): - with module.Runtime_directory({}, 'prefix') as borgmatic_runtime_directory: # noqa: F841 + with module.Runtime_directory({}) as borgmatic_runtime_directory: # noqa: F841 pass @@ -204,7 +204,7 @@ def test_runtime_directory_falls_back_to_temp_and_adds_temporary_subdirectory_th ).and_return(temporary_directory) flexmock(module.os).should_receive('makedirs') - with module.Runtime_directory({}, 'prefix') as borgmatic_runtime_directory: + with module.Runtime_directory({}) as borgmatic_runtime_directory: assert borgmatic_runtime_directory == '/run/borgmatic-1234/./borgmatic' @@ -219,7 +219,7 @@ def test_runtime_directory_with_relative_temp_errors(): flexmock(module.os).should_receive('makedirs') with pytest.raises(ValueError): - with module.Runtime_directory({}, 'prefix') as borgmatic_runtime_directory: # noqa: F841 + with module.Runtime_directory({}) as borgmatic_runtime_directory: # noqa: F841 pass @@ -238,7 +238,7 @@ def test_runtime_directory_falls_back_to_hard_coded_tmp_path_and_adds_temporary_ ).and_return(temporary_directory) flexmock(module.os).should_receive('makedirs') - with module.Runtime_directory({}, 'prefix') as borgmatic_runtime_directory: + with module.Runtime_directory({}) as borgmatic_runtime_directory: assert borgmatic_runtime_directory == '/tmp/borgmatic-1234/./borgmatic' @@ -257,7 +257,7 @@ def test_runtime_directory_with_erroring_cleanup_does_not_raise(): ).and_return(temporary_directory) flexmock(module.os).should_receive('makedirs') - with module.Runtime_directory({}, 'prefix') as borgmatic_runtime_directory: + with module.Runtime_directory({}) as borgmatic_runtime_directory: assert borgmatic_runtime_directory == '/tmp/borgmatic-1234/./borgmatic' diff --git a/tests/unit/hooks/data_source/test_dump.py b/tests/unit/hooks/data_source/test_dump.py index ffc7819c..d1b48ede 100644 --- a/tests/unit/hooks/data_source/test_dump.py +++ b/tests/unit/hooks/data_source/test_dump.py @@ -48,21 +48,21 @@ def test_remove_data_source_dumps_removes_dump_path(): flexmock(module.os.path).should_receive('exists').and_return(True) flexmock(module.shutil).should_receive('rmtree').with_args('databases').once() - module.remove_data_source_dumps('databases', 'SuperDB', 'test.yaml', dry_run=False) + module.remove_data_source_dumps('databases', 'SuperDB', dry_run=False) def test_remove_data_source_dumps_with_dry_run_skips_removal(): flexmock(module.os.path).should_receive('exists').never() flexmock(module.shutil).should_receive('rmtree').never() - module.remove_data_source_dumps('databases', 'SuperDB', 'test.yaml', dry_run=True) + module.remove_data_source_dumps('databases', 'SuperDB', dry_run=True) def test_remove_data_source_dumps_without_dump_path_present_skips_removal(): flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module.shutil).should_receive('rmtree').never() - module.remove_data_source_dumps('databases', 'SuperDB', 'test.yaml', dry_run=False) + module.remove_data_source_dumps('databases', 'SuperDB', dry_run=False) def test_convert_glob_patterns_to_borg_pattern_makes_multipart_regular_expression(): diff --git a/tests/unit/hooks/data_source/test_mariadb.py b/tests/unit/hooks/data_source/test_mariadb.py index 2e4bec78..e21979d6 100644 --- a/tests/unit/hooks/data_source/test_mariadb.py +++ b/tests/unit/hooks/data_source/test_mariadb.py @@ -68,7 +68,6 @@ def test_dump_data_sources_dumps_each_database(): module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -98,7 +97,6 @@ def test_dump_data_sources_dumps_with_password(): assert module.dump_data_sources( [database], {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -123,7 +121,6 @@ def test_dump_data_sources_dumps_all_databases_at_once(): assert module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -151,7 +148,6 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -175,7 +171,7 @@ def test_database_names_to_dump_runs_mariadb_with_list_options(): extra_environment=None, ).and_return(('foo\nbar')).once() - assert module.database_names_to_dump(database, None, 'test.yaml', '') == ('foo', 'bar') + assert module.database_names_to_dump(database, None, '') == ('foo', 'bar') def test_database_names_to_dump_runs_non_default_mariadb_with_list_options(): @@ -196,7 +192,7 @@ def test_database_names_to_dump_runs_non_default_mariadb_with_list_options(): ), ).and_return(('foo\nbar')).once() - assert module.database_names_to_dump(database, None, 'test.yaml', '') == ('foo', 'bar') + assert module.database_names_to_dump(database, None, '') == ('foo', 'bar') def test_execute_dump_command_runs_mariadb_dump(): @@ -455,7 +451,6 @@ def test_dump_data_sources_errors_for_missing_all_databases(): assert module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -475,7 +470,6 @@ def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -500,7 +494,6 @@ def test_restore_data_source_dump_runs_mariadb_to_restore(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source={'name': 'foo'}, dry_run=False, extract_process=extract_process, @@ -529,7 +522,6 @@ def test_restore_data_source_dump_runs_mariadb_with_options(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -560,7 +552,6 @@ def test_restore_data_source_dump_runs_non_default_mariadb_with_options(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -598,7 +589,6 @@ def test_restore_data_source_dump_runs_mariadb_with_hostname_and_port(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -627,7 +617,6 @@ def test_restore_data_source_dump_runs_mariadb_with_username_and_password(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -677,7 +666,6 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -729,7 +717,6 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -751,7 +738,6 @@ def test_restore_data_source_dump_with_dry_run_skips_restore(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source={'name': 'foo'}, dry_run=True, extract_process=flexmock(), diff --git a/tests/unit/hooks/data_source/test_mongodb.py b/tests/unit/hooks/data_source/test_mongodb.py index 0e3efd5d..1d2c0c7b 100644 --- a/tests/unit/hooks/data_source/test_mongodb.py +++ b/tests/unit/hooks/data_source/test_mongodb.py @@ -43,7 +43,6 @@ def test_dump_data_sources_runs_mongodump_for_each_database(): module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -66,7 +65,6 @@ def test_dump_data_sources_with_dry_run_skips_mongodump(): module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -105,7 +103,6 @@ def test_dump_data_sources_runs_mongodump_with_hostname_and_port(): assert module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -151,7 +148,6 @@ def test_dump_data_sources_runs_mongodump_with_username_and_password(): assert module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -177,7 +173,6 @@ def test_dump_data_sources_runs_mongodump_with_directory_format(): module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -213,7 +208,6 @@ def test_dump_data_sources_runs_mongodump_with_options(): assert module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -239,7 +233,6 @@ def test_dump_data_sources_runs_mongodumpall_for_all_databases(): assert module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -271,7 +264,6 @@ def test_restore_data_source_dump_runs_mongorestore(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source={'name': 'foo'}, dry_run=False, extract_process=extract_process, @@ -311,7 +303,6 @@ def test_restore_data_source_dump_runs_mongorestore_with_hostname_and_port(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -359,7 +350,6 @@ def test_restore_data_source_dump_runs_mongorestore_with_username_and_password() module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -415,7 +405,6 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -471,7 +460,6 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -501,7 +489,6 @@ def test_restore_data_source_dump_runs_mongorestore_with_options(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -539,7 +526,6 @@ def test_restore_databases_dump_runs_mongorestore_with_schemas(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -569,7 +555,6 @@ def test_restore_data_source_dump_runs_psql_for_all_database_dump(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -593,7 +578,6 @@ def test_restore_data_source_dump_with_dry_run_skips_restore(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source={'name': 'foo'}, dry_run=True, extract_process=flexmock(), @@ -622,7 +606,6 @@ def test_restore_data_source_dump_without_extract_process_restores_from_disk(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source={'name': 'foo'}, dry_run=False, extract_process=None, diff --git a/tests/unit/hooks/data_source/test_mysql.py b/tests/unit/hooks/data_source/test_mysql.py index a056e7e5..5eb46e35 100644 --- a/tests/unit/hooks/data_source/test_mysql.py +++ b/tests/unit/hooks/data_source/test_mysql.py @@ -68,7 +68,6 @@ def test_dump_data_sources_dumps_each_database(): module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -98,7 +97,6 @@ def test_dump_data_sources_dumps_with_password(): assert module.dump_data_sources( [database], {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -123,7 +121,6 @@ def test_dump_data_sources_dumps_all_databases_at_once(): assert module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -151,7 +148,6 @@ def test_dump_data_sources_dumps_all_databases_separately_when_format_configured module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -175,7 +171,7 @@ def test_database_names_to_dump_runs_mysql_with_list_options(): extra_environment=None, ).and_return(('foo\nbar')).once() - assert module.database_names_to_dump(database, None, 'test.yaml', '') == ('foo', 'bar') + assert module.database_names_to_dump(database, None, '') == ('foo', 'bar') def test_database_names_to_dump_runs_non_default_mysql_with_list_options(): @@ -196,7 +192,7 @@ def test_database_names_to_dump_runs_non_default_mysql_with_list_options(): ), ).and_return(('foo\nbar')).once() - assert module.database_names_to_dump(database, None, 'test.yaml', '') == ('foo', 'bar') + assert module.database_names_to_dump(database, None, '') == ('foo', 'bar') def test_execute_dump_command_runs_mysqldump(): @@ -453,7 +449,6 @@ def test_dump_data_sources_errors_for_missing_all_databases(): assert module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -473,7 +468,6 @@ def test_dump_data_sources_does_not_error_for_missing_all_databases_with_dry_run module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -498,7 +492,6 @@ def test_restore_data_source_dump_runs_mysql_to_restore(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source={'name': 'foo'}, dry_run=False, extract_process=extract_process, @@ -527,7 +520,6 @@ def test_restore_data_source_dump_runs_mysql_with_options(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -556,7 +548,6 @@ def test_restore_data_source_dump_runs_non_default_mysql_with_options(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -594,7 +585,6 @@ def test_restore_data_source_dump_runs_mysql_with_hostname_and_port(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -623,7 +613,6 @@ def test_restore_data_source_dump_runs_mysql_with_username_and_password(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -673,7 +662,6 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source={'name': 'foo'}, dry_run=False, extract_process=extract_process, @@ -725,7 +713,6 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -747,7 +734,6 @@ def test_restore_data_source_dump_with_dry_run_skips_restore(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source={'name': 'foo'}, dry_run=True, extract_process=flexmock(), diff --git a/tests/unit/hooks/data_source/test_postgresql.py b/tests/unit/hooks/data_source/test_postgresql.py index 752f940b..4188f96d 100644 --- a/tests/unit/hooks/data_source/test_postgresql.py +++ b/tests/unit/hooks/data_source/test_postgresql.py @@ -61,7 +61,7 @@ def test_make_extra_environment_without_ssl_mode_does_not_set_ssl_mode(): def test_database_names_to_dump_passes_through_individual_database_name(): database = {'name': 'foo'} - assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == ( + assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ( 'foo', ) @@ -69,7 +69,7 @@ def test_database_names_to_dump_passes_through_individual_database_name(): def test_database_names_to_dump_passes_through_individual_database_name_with_format(): database = {'name': 'foo', 'format': 'custom'} - assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == ( + assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ( 'foo', ) @@ -77,7 +77,7 @@ def test_database_names_to_dump_passes_through_individual_database_name_with_for def test_database_names_to_dump_passes_through_all_without_format(): database = {'name': 'all'} - assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == ( + assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ( 'all', ) @@ -86,7 +86,7 @@ def test_database_names_to_dump_with_all_and_format_and_dry_run_bails(): database = {'name': 'all', 'format': 'custom'} flexmock(module).should_receive('execute_command_and_capture_output').never() - assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=True) == () + assert module.database_names_to_dump(database, flexmock(), dry_run=True) == () def test_database_names_to_dump_with_all_and_format_lists_databases(): @@ -95,7 +95,7 @@ def test_database_names_to_dump_with_all_and_format_lists_databases(): 'foo,test,\nbar,test,"stuff and such"' ) - assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == ( + assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ( 'foo', 'bar', ) @@ -119,7 +119,7 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostnam extra_environment=object, ).and_return('foo,test,\nbar,test,"stuff and such"') - assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == ( + assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ( 'foo', 'bar', ) @@ -141,7 +141,7 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_usernam extra_environment=object, ).and_return('foo,test,\nbar,test,"stuff and such"') - assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == ( + assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ( 'foo', 'bar', ) @@ -154,7 +154,7 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_options extra_environment=object, ).and_return('foo,test,\nbar,test,"stuff and such"') - assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == ( + assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ( 'foo', 'bar', ) @@ -166,7 +166,7 @@ def test_database_names_to_dump_with_all_and_format_excludes_particular_database 'foo,test,\ntemplate0,test,blah' ) - assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == ( + assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ( 'foo', ) @@ -194,7 +194,7 @@ def test_database_names_to_dump_with_all_and_psql_command_uses_custom_command(): extra_environment=object, ).and_return('foo,text').once() - assert module.database_names_to_dump(database, flexmock(), flexmock(), dry_run=False) == ( + assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ( 'foo', ) @@ -253,7 +253,6 @@ def test_dump_data_sources_runs_pg_dump_for_each_database(): module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -273,7 +272,6 @@ def test_dump_data_sources_raises_when_no_database_names_to_dump(): module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -290,7 +288,6 @@ def test_dump_data_sources_does_not_raise_when_no_database_names_to_dump(): module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -316,7 +313,6 @@ def test_dump_data_sources_with_duplicate_dump_skips_pg_dump(): module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -344,7 +340,6 @@ def test_dump_data_sources_with_dry_run_skips_pg_dump(): module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -390,7 +385,6 @@ def test_dump_data_sources_runs_pg_dump_with_hostname_and_port(): assert module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -434,7 +428,6 @@ def test_dump_data_sources_runs_pg_dump_with_username_and_password(): assert module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -478,7 +471,6 @@ def test_dump_data_sources_with_username_injection_attack_gets_escaped(): assert module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -518,7 +510,6 @@ def test_dump_data_sources_runs_pg_dump_with_directory_format(): module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -561,7 +552,6 @@ def test_dump_data_sources_runs_pg_dump_with_options(): assert module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -591,7 +581,6 @@ def test_dump_data_sources_runs_pg_dumpall_for_all_databases(): assert module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -633,7 +622,6 @@ def test_dump_data_sources_runs_non_default_pg_dump(): assert module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -680,7 +668,6 @@ def test_restore_data_source_dump_runs_pg_restore(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source={'name': 'foo'}, dry_run=False, extract_process=extract_process, @@ -743,7 +730,6 @@ def test_restore_data_source_dump_runs_pg_restore_with_hostname_and_port(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -804,7 +790,6 @@ def test_restore_data_source_dump_runs_pg_restore_with_username_and_password(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -884,7 +869,6 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source={'name': 'foo'}, dry_run=False, extract_process=extract_process, @@ -964,7 +948,6 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -1026,7 +1009,6 @@ def test_restore_data_source_dump_runs_pg_restore_with_options(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -1066,7 +1048,6 @@ def test_restore_data_source_dump_runs_psql_for_all_database_dump(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source={'name': 'all'}, dry_run=False, extract_process=extract_process, @@ -1111,7 +1092,6 @@ def test_restore_data_source_dump_runs_psql_for_plain_database_dump(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -1181,7 +1161,6 @@ def test_restore_data_source_dump_runs_non_default_pg_restore_and_psql(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -1206,7 +1185,6 @@ def test_restore_data_source_dump_with_dry_run_skips_restore(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source={'name': 'foo'}, dry_run=True, extract_process=flexmock(), @@ -1259,7 +1237,6 @@ def test_restore_data_source_dump_without_extract_process_restores_from_disk(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source={'name': 'foo'}, dry_run=False, extract_process=None, @@ -1316,7 +1293,6 @@ def test_restore_data_source_dump_with_schemas_restores_schemas(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=None, diff --git a/tests/unit/hooks/data_source/test_sqlite.py b/tests/unit/hooks/data_source/test_sqlite.py index 02c343a3..03afa3c3 100644 --- a/tests/unit/hooks/data_source/test_sqlite.py +++ b/tests/unit/hooks/data_source/test_sqlite.py @@ -31,7 +31,6 @@ def test_dump_data_sources_logs_and_skips_if_dump_already_exists(): module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -62,7 +61,6 @@ def test_dump_data_sources_dumps_each_database(): module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -100,7 +98,6 @@ def test_dump_data_sources_with_path_injection_attack_gets_escaped(): module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -129,7 +126,6 @@ def test_dump_data_sources_with_non_existent_path_warns_and_dumps_database(): module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -160,7 +156,6 @@ def test_dump_data_sources_with_name_all_warns_and_dumps_all_databases(): module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -185,7 +180,6 @@ def test_dump_data_sources_does_not_dump_if_dry_run(): module.dump_data_sources( databases, {}, - 'test.yaml', config_paths=('test.yaml',), borgmatic_runtime_directory='/run/borgmatic', patterns=[], @@ -214,7 +208,6 @@ def test_restore_data_source_dump_restores_database(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -244,7 +237,6 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_ module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source={'name': 'database'}, dry_run=False, extract_process=extract_process, @@ -274,7 +266,6 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_ module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source=hook_config[0], dry_run=False, extract_process=extract_process, @@ -293,7 +284,6 @@ def test_restore_data_source_dump_does_not_restore_database_if_dry_run(): module.restore_data_source_dump( hook_config, {}, - 'test.yaml', data_source={'name': 'database'}, dry_run=True, extract_process=extract_process, diff --git a/tests/unit/hooks/test_dispatch.py b/tests/unit/hooks/test_dispatch.py index dde8bd5d..d0eca83d 100644 --- a/tests/unit/hooks/test_dispatch.py +++ b/tests/unit/hooks/test_dispatch.py @@ -27,10 +27,10 @@ def test_call_hook_invokes_module_function_with_arguments_and_returns_value(): 'borgmatic.hooks.monitoring.super_hook' ).and_return(test_module) flexmock(test_module).should_receive('hook_function').with_args( - config['super_hook'], config, 'prefix', 55, value=66 + config['super_hook'], config, 55, value=66 ).and_return(expected_return_value).once() - return_value = module.call_hook('hook_function', config, 'prefix', 'super_hook', 55, value=66) + return_value = module.call_hook('hook_function', config, 'super_hook', 55, value=66) assert return_value == expected_return_value @@ -49,10 +49,10 @@ def test_call_hook_probes_config_with_databases_suffix(): 'borgmatic.hooks.monitoring.super_hook' ).and_return(test_module) flexmock(test_module).should_receive('hook_function').with_args( - config['super_hook_databases'], config, 'prefix', 55, value=66 + config['super_hook_databases'], config, 55, value=66 ).and_return(expected_return_value).once() - return_value = module.call_hook('hook_function', config, 'prefix', 'super_hook', 55, value=66) + return_value = module.call_hook('hook_function', config, 'super_hook', 55, value=66) assert return_value == expected_return_value @@ -71,11 +71,11 @@ def test_call_hook_strips_databases_suffix_from_hook_name(): 'borgmatic.hooks.monitoring.super_hook' ).and_return(test_module) flexmock(test_module).should_receive('hook_function').with_args( - config['super_hook_databases'], config, 'prefix', 55, value=66 + config['super_hook_databases'], config, 55, value=66 ).and_return(expected_return_value).once() return_value = module.call_hook( - 'hook_function', config, 'prefix', 'super_hook_databases', 55, value=66 + 'hook_function', config, 'super_hook_databases', 55, value=66 ) assert return_value == expected_return_value @@ -95,10 +95,10 @@ def test_call_hook_without_hook_config_invokes_module_function_with_arguments_an 'borgmatic.hooks.monitoring.super_hook' ).and_return(test_module) flexmock(test_module).should_receive('hook_function').with_args( - None, config, 'prefix', 55, value=66 + None, config, 55, value=66 ).and_return(expected_return_value).once() - return_value = module.call_hook('hook_function', config, 'prefix', 'super_hook', 55, value=66) + return_value = module.call_hook('hook_function', config, 'super_hook', 55, value=66) assert return_value == expected_return_value @@ -118,7 +118,7 @@ def test_call_hook_without_corresponding_module_raises(): flexmock(test_module).should_receive('hook_function').never() with pytest.raises(ValueError): - module.call_hook('hook_function', config, 'prefix', 'super_hook', 55, value=66) + module.call_hook('hook_function', config, 'super_hook', 55, value=66) def test_call_hook_skips_non_hook_modules(): @@ -134,7 +134,7 @@ def test_call_hook_skips_non_hook_modules(): 'borgmatic.hooks.monitoring.not_a_hook' ).and_return(not_a_hook_module) - return_value = module.call_hook('hook_function', config, 'prefix', 'not_a_hook', 55, value=66) + return_value = module.call_hook('hook_function', config, 'not_a_hook', 55, value=66) assert return_value is None @@ -152,7 +152,7 @@ def test_call_hooks_calls_each_hook_and_collects_return_values(): expected_return_values['super_hook'] ).and_return(expected_return_values['other_hook']) - return_values = module.call_hooks('do_stuff', config, 'prefix', module.Hook_type.MONITORING, 55) + return_values = module.call_hooks('do_stuff', config, module.Hook_type.MONITORING, 55) assert return_values == expected_return_values @@ -168,7 +168,7 @@ def test_call_hooks_calls_skips_return_values_for_unconfigured_hooks(): ).and_return(['super_hook', 'other_hook']) flexmock(module).should_receive('call_hook').and_return(expected_return_values['super_hook']) - return_values = module.call_hooks('do_stuff', config, 'prefix', module.Hook_type.MONITORING, 55) + return_values = module.call_hooks('do_stuff', config, module.Hook_type.MONITORING, 55) assert return_values == expected_return_values @@ -186,7 +186,7 @@ def test_call_hooks_calls_treats_null_hook_as_optionless(): expected_return_values['super_hook'] ).and_return(expected_return_values['other_hook']) - return_values = module.call_hooks('do_stuff', config, 'prefix', module.Hook_type.MONITORING, 55) + return_values = module.call_hooks('do_stuff', config, module.Hook_type.MONITORING, 55) assert return_values == expected_return_values @@ -204,7 +204,7 @@ def test_call_hooks_calls_looks_up_databases_suffix_in_config(): expected_return_values['super_hook'] ).and_return(expected_return_values['other_hook']) - return_values = module.call_hooks('do_stuff', config, 'prefix', module.Hook_type.MONITORING, 55) + return_values = module.call_hooks('do_stuff', config, module.Hook_type.MONITORING, 55) assert return_values == expected_return_values @@ -223,7 +223,7 @@ def test_call_hooks_even_if_unconfigured_calls_each_hook_and_collects_return_val ).and_return(expected_return_values['other_hook']) return_values = module.call_hooks_even_if_unconfigured( - 'do_stuff', config, 'prefix', module.Hook_type.MONITORING, 55 + 'do_stuff', config, module.Hook_type.MONITORING, 55 ) assert return_values == expected_return_values @@ -243,7 +243,7 @@ def test_call_hooks_even_if_unconfigured_calls_each_hook_configured_or_not_and_c ).and_return(expected_return_values['other_hook']) return_values = module.call_hooks_even_if_unconfigured( - 'do_stuff', config, 'prefix', module.Hook_type.MONITORING, 55 + 'do_stuff', config, module.Hook_type.MONITORING, 55 ) assert return_values == expected_return_values diff --git a/tests/unit/test_execute.py b/tests/unit/test_execute.py index 82c94206..7741526f 100644 --- a/tests/unit/test_execute.py +++ b/tests/unit/test_execute.py @@ -186,6 +186,7 @@ def test_execute_command_calls_full_command(): env=None, cwd=None, ).and_return(flexmock(stdout=None)).once() + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') output = module.execute_command(full_command) @@ -207,6 +208,7 @@ def test_execute_command_calls_full_command_with_output_file(): env=None, cwd=None, ).and_return(flexmock(stderr=None)).once() + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') output = module.execute_command(full_command, output_file=output_file) @@ -222,6 +224,7 @@ def test_execute_command_calls_full_command_without_capturing_output(): full_command, stdin=None, stdout=None, stderr=None, shell=False, env=None, cwd=None ).and_return(flexmock(wait=lambda: 0)).once() flexmock(module).should_receive('interpret_exit_code').and_return(module.Exit_status.SUCCESS) + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') output = module.execute_command(full_command, output_file=module.DO_NOT_CAPTURE) @@ -243,6 +246,7 @@ def test_execute_command_calls_full_command_with_input_file(): env=None, cwd=None, ).and_return(flexmock(stdout=None)).once() + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') output = module.execute_command(full_command, input_file=input_file) @@ -263,6 +267,7 @@ def test_execute_command_calls_full_command_with_shell(): env=None, cwd=None, ).and_return(flexmock(stdout=None)).once() + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') output = module.execute_command(full_command, shell=True) @@ -283,6 +288,7 @@ def test_execute_command_calls_full_command_with_extra_environment(): env={'a': 'b', 'c': 'd'}, cwd=None, ).and_return(flexmock(stdout=None)).once() + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') output = module.execute_command(full_command, extra_environment={'c': 'd'}) @@ -303,6 +309,7 @@ def test_execute_command_calls_full_command_with_working_directory(): env=None, cwd='/working', ).and_return(flexmock(stdout=None)).once() + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') output = module.execute_command(full_command, working_directory='/working') @@ -324,6 +331,7 @@ def test_execute_command_without_run_to_completion_returns_process(): env=None, cwd=None, ).and_return(process).once() + flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock()) flexmock(module).should_receive('log_outputs') assert module.execute_command(full_command, run_to_completion=False) == process diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index cc6619ed..97aeb63a 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -177,8 +177,8 @@ def test_multi_stream_handler_logs_to_handler_for_log_level(): def test_console_color_formatter_format_includes_log_message(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.ANSWER - flexmock(module.logging.Formatter).should_receive('format') plain_message = 'uh oh' + flexmock(module.logging.Formatter).should_receive('format').and_return(plain_message) record = flexmock(levelno=logging.CRITICAL) colored_message = module.Console_color_formatter().format(record) From 31b6e2113958c91dcd321d005f12e2f71d60e572 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 26 Jan 2025 19:03:40 -0800 Subject: [PATCH 07/12] Fix end-to-end tests and update more log messages (#635). --- borgmatic/commands/borgmatic.py | 5 +++ borgmatic/hooks/monitoring/apprise.py | 8 ++-- borgmatic/hooks/monitoring/cronhub.py | 10 ++--- borgmatic/hooks/monitoring/cronitor.py | 10 ++--- borgmatic/hooks/monitoring/healthchecks.py | 12 +++--- borgmatic/hooks/monitoring/loki.py | 4 +- borgmatic/hooks/monitoring/ntfy.py | 16 ++++---- borgmatic/hooks/monitoring/pagerduty.py | 10 ++--- borgmatic/hooks/monitoring/pushover.py | 10 ++--- borgmatic/hooks/monitoring/sentry.py | 14 +++---- borgmatic/hooks/monitoring/uptime_kuma.py | 8 ++-- borgmatic/hooks/monitoring/zabbix.py | 32 +++++++-------- borgmatic/logger.py | 40 +++++++++++++------ tests/end-to-end/test_override.py | 2 +- .../hooks/monitoring/test_apprise.py | 4 +- .../hooks/monitoring/test_healthchecks.py | 4 +- .../integration/hooks/monitoring/test_loki.py | 6 +-- tests/unit/hooks/monitoring/test_apprise.py | 1 - 18 files changed, 108 insertions(+), 88 deletions(-) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 112a7691..1bcdbb7f 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -108,6 +108,7 @@ def run_configuration(config_filename, config, config_paths, arguments): 'initialize_monitor', config, dispatch.Hook_type.MONITORING, + config_filename, monitoring_log_level, global_arguments.dry_run, ) @@ -116,6 +117,7 @@ def run_configuration(config_filename, config, config_paths, arguments): 'ping_monitor', config, dispatch.Hook_type.MONITORING, + config_filename, monitor.State.START, monitoring_log_level, global_arguments.dry_run, @@ -187,6 +189,7 @@ def run_configuration(config_filename, config, config_paths, arguments): 'ping_monitor', config, dispatch.Hook_type.MONITORING, + config_filename, monitor.State.LOG, monitoring_log_level, global_arguments.dry_run, @@ -203,6 +206,7 @@ def run_configuration(config_filename, config, config_paths, arguments): 'ping_monitor', config, dispatch.Hook_type.MONITORING, + config_filename, monitor.State.FINISH, monitoring_log_level, global_arguments.dry_run, @@ -237,6 +241,7 @@ def run_configuration(config_filename, config, config_paths, arguments): 'ping_monitor', config, dispatch.Hook_type.MONITORING, + config_filename, monitor.State.FAIL, monitoring_log_level, global_arguments.dry_run, diff --git a/borgmatic/hooks/monitoring/apprise.py b/borgmatic/hooks/monitoring/apprise.py index 9592d787..b873c6cd 100644 --- a/borgmatic/hooks/monitoring/apprise.py +++ b/borgmatic/hooks/monitoring/apprise.py @@ -66,12 +66,12 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev ) if not hook_config.get('services'): - logger.info(f'{config_filename}: No Apprise services to ping') + logger.info('No Apprise services to ping') return dry_run_string = ' (dry run; not actually pinging)' if dry_run else '' labels_string = ', '.join(map(operator.itemgetter('label'), hook_config.get('services'))) - logger.info(f'{config_filename}: Pinging Apprise services: {labels_string}{dry_run_string}') + logger.info(f'Pinging Apprise services: {labels_string}{dry_run_string}') apprise_object = apprise.Apprise() apprise_object.add(list(map(operator.itemgetter('url'), hook_config.get('services')))) @@ -100,10 +100,10 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev ) if result is False: - logger.warning(f'{config_filename}: Error sending some Apprise notifications') + logger.warning('Error sending some Apprise notifications') -def destroy_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run): +def destroy_monitor(hook_config, config, monitoring_log_level, dry_run): ''' Remove the monitor handler that was added to the root logger. This prevents the handler from getting reused by other instances of this monitor. diff --git a/borgmatic/hooks/monitoring/cronhub.py b/borgmatic/hooks/monitoring/cronhub.py index fbb84c8b..dfc4892f 100644 --- a/borgmatic/hooks/monitoring/cronhub.py +++ b/borgmatic/hooks/monitoring/cronhub.py @@ -29,7 +29,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev ''' if state not in MONITOR_STATE_TO_CRONHUB: logger.debug( - f'{config_filename}: Ignoring unsupported monitoring {state.name.lower()} in Cronhub hook' + f'Ignoring unsupported monitoring {state.name.lower()} in Cronhub hook' ) return @@ -41,8 +41,8 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev .replace('/ping/', formatted_state) ) - logger.info(f'{config_filename}: Pinging Cronhub {state.name.lower()}{dry_run_label}') - logger.debug(f'{config_filename}: Using Cronhub ping URL {ping_url}') + logger.info(f'Pinging Cronhub {state.name.lower()}{dry_run_label}') + logger.debug(f'Using Cronhub ping URL {ping_url}') if not dry_run: logging.getLogger('urllib3').setLevel(logging.ERROR) @@ -51,11 +51,11 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev if not response.ok: response.raise_for_status() except requests.exceptions.RequestException as error: - logger.warning(f'{config_filename}: Cronhub error: {error}') + logger.warning(f'Cronhub error: {error}') def destroy_monitor( - ping_url_or_uuid, config, config_filename, monitoring_log_level, dry_run + ping_url_or_uuid, config, monitoring_log_level, dry_run ): # pragma: no cover ''' No destruction is necessary for this monitor. diff --git a/borgmatic/hooks/monitoring/cronitor.py b/borgmatic/hooks/monitoring/cronitor.py index 87a2072e..994dab31 100644 --- a/borgmatic/hooks/monitoring/cronitor.py +++ b/borgmatic/hooks/monitoring/cronitor.py @@ -29,15 +29,15 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev ''' if state not in MONITOR_STATE_TO_CRONITOR: logger.debug( - f'{config_filename}: Ignoring unsupported monitoring {state.name.lower()} in Cronitor hook' + f'Ignoring unsupported monitoring {state.name.lower()} in Cronitor hook' ) return dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' ping_url = f"{hook_config['ping_url']}/{MONITOR_STATE_TO_CRONITOR[state]}" - logger.info(f'{config_filename}: Pinging Cronitor {state.name.lower()}{dry_run_label}') - logger.debug(f'{config_filename}: Using Cronitor ping URL {ping_url}') + logger.info(f'Pinging Cronitor {state.name.lower()}{dry_run_label}') + logger.debug(f'Using Cronitor ping URL {ping_url}') if not dry_run: logging.getLogger('urllib3').setLevel(logging.ERROR) @@ -46,11 +46,11 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev if not response.ok: response.raise_for_status() except requests.exceptions.RequestException as error: - logger.warning(f'{config_filename}: Cronitor error: {error}') + logger.warning(f'Cronitor error: {error}') def destroy_monitor( - ping_url_or_uuid, config, config_filename, monitoring_log_level, dry_run + ping_url_or_uuid, config, monitoring_log_level, dry_run ): # pragma: no cover ''' No destruction is necessary for this monitor. diff --git a/borgmatic/hooks/monitoring/healthchecks.py b/borgmatic/hooks/monitoring/healthchecks.py index 8483f98b..1e145106 100644 --- a/borgmatic/hooks/monitoring/healthchecks.py +++ b/borgmatic/hooks/monitoring/healthchecks.py @@ -56,7 +56,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev if 'states' in hook_config and state.name.lower() not in hook_config['states']: logger.info( - f'{config_filename}: Skipping Healthchecks {state.name.lower()} ping due to configured states' + f'Skipping Healthchecks {state.name.lower()} ping due to configured states' ) return @@ -69,13 +69,13 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev if hook_config.get('create_slug'): if ping_url_is_uuid: logger.warning( - f'{config_filename}: Healthchecks UUIDs do not support auto provisionning; ignoring' + 'Healthchecks UUIDs do not support auto provisionning; ignoring' ) else: ping_url = f'{ping_url}?create=1' - logger.info(f'{config_filename}: Pinging Healthchecks {state.name.lower()}{dry_run_label}') - logger.debug(f'{config_filename}: Using Healthchecks ping URL {ping_url}') + logger.info(f'Pinging Healthchecks {state.name.lower()}{dry_run_label}') + logger.debug(f'Using Healthchecks ping URL {ping_url}') if state in (monitor.State.FINISH, monitor.State.FAIL, monitor.State.LOG): payload = borgmatic.hooks.monitoring.logs.format_buffered_logs_for_payload( @@ -93,10 +93,10 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev if not response.ok: response.raise_for_status() except requests.exceptions.RequestException as error: - logger.warning(f'{config_filename}: Healthchecks error: {error}') + logger.warning(f'Healthchecks error: {error}') -def destroy_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run): +def destroy_monitor(hook_config, config, monitoring_log_level, dry_run): ''' Remove the monitor handler that was added to the root logger. This prevents the handler from getting reused by other instances of this monitor. diff --git a/borgmatic/hooks/monitoring/loki.py b/borgmatic/hooks/monitoring/loki.py index 51e287c7..2c0405b1 100644 --- a/borgmatic/hooks/monitoring/loki.py +++ b/borgmatic/hooks/monitoring/loki.py @@ -139,10 +139,10 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev for handler in tuple(logging.getLogger().handlers): if isinstance(handler, Loki_log_handler): if state in MONITOR_STATE_TO_LOKI.keys(): - handler.raw(f'{config_filename}: {MONITOR_STATE_TO_LOKI[state]} backup') + handler.raw(f'{MONITOR_STATE_TO_LOKI[state]} backup') -def destroy_monitor(hook_config, config, config_filename, monitoring_log_level, dry_run): +def destroy_monitor(hook_config, config, monitoring_log_level, dry_run): ''' Remove the monitor handler that was added to the root logger. ''' diff --git a/borgmatic/hooks/monitoring/ntfy.py b/borgmatic/hooks/monitoring/ntfy.py index d3a37ca6..9fa28c80 100644 --- a/borgmatic/hooks/monitoring/ntfy.py +++ b/borgmatic/hooks/monitoring/ntfy.py @@ -37,8 +37,8 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev base_url = hook_config.get('server', 'https://ntfy.sh') topic = hook_config.get('topic') - logger.info(f'{config_filename}: Pinging ntfy topic {topic}{dry_run_label}') - logger.debug(f'{config_filename}: Using Ntfy ping URL {base_url}/{topic}') + logger.info(f'Pinging ntfy topic {topic}{dry_run_label}') + logger.debug(f'Using Ntfy ping URL {base_url}/{topic}') headers = { 'X-Title': state_config.get('title'), @@ -55,19 +55,19 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev if access_token is not None: if username or password: logger.warning( - f'{config_filename}: ntfy access_token is set but so is username/password, only using access_token' + 'ntfy access_token is set but so is username/password, only using access_token' ) auth = requests.auth.HTTPBasicAuth('', access_token) elif (username and password) is not None: auth = requests.auth.HTTPBasicAuth(username, password) - logger.info(f'{config_filename}: Using basic auth with user {username} for ntfy') + logger.info(f'Using basic auth with user {username} for ntfy') elif username is not None: logger.warning( - f'{config_filename}: Password missing for ntfy authentication, defaulting to no auth' + 'Password missing for ntfy authentication, defaulting to no auth' ) elif password is not None: logger.warning( - f'{config_filename}: Username missing for ntfy authentication, defaulting to no auth' + 'Username missing for ntfy authentication, defaulting to no auth' ) if not dry_run: @@ -77,11 +77,11 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev if not response.ok: response.raise_for_status() except requests.exceptions.RequestException as error: - logger.warning(f'{config_filename}: ntfy error: {error}') + logger.warning(f'ntfy error: {error}') def destroy_monitor( - ping_url_or_uuid, config, config_filename, monitoring_log_level, dry_run + ping_url_or_uuid, config, monitoring_log_level, dry_run ): # pragma: no cover ''' No destruction is necessary for this monitor. diff --git a/borgmatic/hooks/monitoring/pagerduty.py b/borgmatic/hooks/monitoring/pagerduty.py index 2d5325de..5e4aea4f 100644 --- a/borgmatic/hooks/monitoring/pagerduty.py +++ b/borgmatic/hooks/monitoring/pagerduty.py @@ -29,12 +29,12 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev ''' if state != monitor.State.FAIL: logger.debug( - f'{config_filename}: Ignoring unsupported monitoring {state.name.lower()} in PagerDuty hook', + f'Ignoring unsupported monitoring {state.name.lower()} in PagerDuty hook', ) return dry_run_label = ' (dry run; not actually sending)' if dry_run else '' - logger.info(f'{config_filename}: Sending failure event to PagerDuty {dry_run_label}') + logger.info(f'Sending failure event to PagerDuty {dry_run_label}') if dry_run: return @@ -61,7 +61,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev }, } ) - logger.debug(f'{config_filename}: Using PagerDuty payload: {payload}') + logger.debug(f'Using PagerDuty payload: {payload}') logging.getLogger('urllib3').setLevel(logging.ERROR) try: @@ -69,11 +69,11 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev if not response.ok: response.raise_for_status() except requests.exceptions.RequestException as error: - logger.warning(f'{config_filename}: PagerDuty error: {error}') + logger.warning(f'PagerDuty error: {error}') def destroy_monitor( - ping_url_or_uuid, config, config_filename, monitoring_log_level, dry_run + ping_url_or_uuid, config, monitoring_log_level, dry_run ): # pragma: no cover ''' No destruction is necessary for this monitor. diff --git a/borgmatic/hooks/monitoring/pushover.py b/borgmatic/hooks/monitoring/pushover.py index 739283bd..40f93d93 100644 --- a/borgmatic/hooks/monitoring/pushover.py +++ b/borgmatic/hooks/monitoring/pushover.py @@ -35,14 +35,14 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev token = hook_config.get('token') user = hook_config.get('user') - logger.info(f'{config_filename}: Updating Pushover{dry_run_label}') + logger.info(f'Updating Pushover{dry_run_label}') if state_config.get('priority') == EMERGENCY_PRIORITY: if 'expire' not in state_config: - logger.info(f'{config_filename}: Setting expire to default (10 min).') + logger.info('Setting expire to default (10 min)') state_config['expire'] = 600 if 'retry' not in state_config: - logger.info(f'{config_filename}: Setting retry to default (30 sec).') + logger.info('Setting retry to default (30 sec)') state_config['retry'] = 30 else: if 'expire' in state_config or 'retry' in state_config: @@ -75,11 +75,11 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev if not response.ok: response.raise_for_status() except requests.exceptions.RequestException as error: - logger.warning(f'{config_filename}: Pushover error: {error}') + logger.warning(f'Pushover error: {error}') def destroy_monitor( - ping_url_or_uuid, config, config_filename, monitoring_log_level, dry_run + ping_url_or_uuid, config, monitoring_log_level, dry_run ): # pragma: no cover ''' No destruction is necessary for this monitor. diff --git a/borgmatic/hooks/monitoring/sentry.py b/borgmatic/hooks/monitoring/sentry.py index 5a561760..dbc8f014 100644 --- a/borgmatic/hooks/monitoring/sentry.py +++ b/borgmatic/hooks/monitoring/sentry.py @@ -16,7 +16,7 @@ def initialize_monitor( DATA_SOURCE_NAME_URL_PATTERN = re.compile( - '^(?P.*)://(?P.*)@(?P.*)/(?P.*)$' + '^(?P.+)://(?P.+)@(?P.+)/(?P.+)$' ) @@ -39,14 +39,14 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev if not match: logger.warning( - 'f{config_filename}: Invalid Sentry data source name URL: {data_source_name_url}' + f'Invalid Sentry data source name URL: {data_source_name_url}' ) return cron_url = f'{match.group("protocol")}://{match.group("hostname")}/api/{match.group("project_id")}/cron/{monitor_slug}/{match.group("username")}/' - logger.info(f'{config_filename}: Pinging Sentry {state.name.lower()}{dry_run_label}') - logger.debug(f'{config_filename}: Using Sentry cron URL {cron_url}') + logger.info(f'Pinging Sentry {state.name.lower()}{dry_run_label}') + logger.debug(f'Using Sentry cron URL {cron_url}') status = { 'start': 'in_progress', @@ -55,7 +55,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev }.get(state.name.lower()) if not status: - logger.warning('f{config_filename}: Invalid Sentry state') + logger.warning('Invalid Sentry state') return if dry_run: @@ -67,11 +67,11 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev if not response.ok: response.raise_for_status() except requests.exceptions.RequestException as error: - logger.warning(f'{config_filename}: Sentry error: {error}') + logger.warning(f'Sentry error: {error}') def destroy_monitor( - ping_url_or_uuid, config, config_filename, monitoring_log_level, dry_run + ping_url_or_uuid, config, monitoring_log_level, dry_run ): # pragma: no cover ''' No destruction is necessary for this monitor. diff --git a/borgmatic/hooks/monitoring/uptime_kuma.py b/borgmatic/hooks/monitoring/uptime_kuma.py index 46fa5247..9b06b750 100644 --- a/borgmatic/hooks/monitoring/uptime_kuma.py +++ b/borgmatic/hooks/monitoring/uptime_kuma.py @@ -29,9 +29,9 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev push_url = hook_config.get('push_url', 'https://example.uptime.kuma/api/push/abcd1234') query = f'status={status}&msg={state.name.lower()}' logger.info( - f'{config_filename}: Pushing Uptime Kuma push_url {push_url}?{query} {dry_run_label}' + f'Pushing Uptime Kuma push_url {push_url}?{query} {dry_run_label}' ) - logger.debug(f'{config_filename}: Full Uptime Kuma state URL {push_url}?{query}') + logger.debug(f'Full Uptime Kuma state URL {push_url}?{query}') if dry_run: return @@ -43,11 +43,11 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev if not response.ok: response.raise_for_status() except requests.exceptions.RequestException as error: - logger.warning(f'{config_filename}: Uptime Kuma error: {error}') + logger.warning(f'Uptime Kuma error: {error}') def destroy_monitor( - push_url_or_uuid, config, config_filename, monitoring_log_level, dry_run + push_url_or_uuid, config, monitoring_log_level, dry_run ): # pragma: no cover ''' No destruction is necessary for this monitor. diff --git a/borgmatic/hooks/monitoring/zabbix.py b/borgmatic/hooks/monitoring/zabbix.py index a33d77b4..17744a21 100644 --- a/borgmatic/hooks/monitoring/zabbix.py +++ b/borgmatic/hooks/monitoring/zabbix.py @@ -44,16 +44,16 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev value = state_config.get('value') headers = {'Content-Type': 'application/json-rpc'} - logger.info(f'{config_filename}: Updating Zabbix{dry_run_label}') - logger.debug(f'{config_filename}: Using Zabbix URL: {server}') + logger.info(f'Updating Zabbix{dry_run_label}') + logger.debug(f'Using Zabbix URL: {server}') if server is None: - logger.warning(f'{config_filename}: Server missing for Zabbix') + logger.warning('Server missing for Zabbix') return # Determine the Zabbix method used to store the value: itemid or host/key if itemid is not None: - logger.info(f'{config_filename}: Updating {itemid} on Zabbix') + logger.info(f'Updating {itemid} on Zabbix') data = { 'jsonrpc': '2.0', 'method': 'history.push', @@ -62,7 +62,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev } elif (host and key) is not None: - logger.info(f'{config_filename}: Updating Host:{host} and Key:{key} on Zabbix') + logger.info(f'Updating Host:{host} and Key:{key} on Zabbix') data = { 'jsonrpc': '2.0', 'method': 'history.push', @@ -71,23 +71,23 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev } elif host is not None: - logger.warning(f'{config_filename}: Key missing for Zabbix') + logger.warning('Key missing for Zabbix') return elif key is not None: - logger.warning(f'{config_filename}: Host missing for Zabbix.') + logger.warning('Host missing for Zabbix') return else: - logger.warning(f'{config_filename}: No zabbix itemid or host/key provided.') + logger.warning('No Zabbix itemid or host/key provided') return # Determine the authentication method: API key or username/password if api_key is not None: - logger.info(f'{config_filename}: Using API key auth for Zabbix') + logger.info('Using API key auth for Zabbix') headers['Authorization'] = 'Bearer ' + api_key elif (username and password) is not None: - logger.info(f'{config_filename}: Using user/pass auth with user {username} for Zabbix') + logger.info('Using user/pass auth with user {username} for Zabbix') auth_data = { 'jsonrpc': '2.0', 'method': 'user.login', @@ -102,18 +102,18 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev if not response.ok: response.raise_for_status() except requests.exceptions.RequestException as error: - logger.warning(f'{config_filename}: Zabbix error: {error}') + logger.warning(f'Zabbix error: {error}') return elif username is not None: - logger.warning(f'{config_filename}: Password missing for Zabbix authentication') + logger.warning('Password missing for Zabbix authentication') return elif password is not None: - logger.warning(f'{config_filename}: Username missing for Zabbix authentication') + logger.warning('Username missing for Zabbix authentication') return else: - logger.warning(f'{config_filename}: Authentication data missing for Zabbix') + logger.warning('Authentication data missing for Zabbix') return if not dry_run: @@ -123,11 +123,11 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev if not response.ok: response.raise_for_status() except requests.exceptions.RequestException as error: - logger.warning(f'{config_filename}: Zabbix error: {error}') + logger.warning(f'Zabbix error: {error}') def destroy_monitor( - ping_url_or_uuid, config, config_filename, monitoring_log_level, dry_run + ping_url_or_uuid, config, monitoring_log_level, dry_run ): # pragma: no cover ''' No destruction is necessary for this monitor. diff --git a/borgmatic/logger.py b/borgmatic/logger.py index a2eceacd..e6203fcd 100644 --- a/borgmatic/logger.py +++ b/borgmatic/logger.py @@ -186,6 +186,31 @@ def add_custom_log_levels(): # pragma: no cover add_logging_level('DISABLED', DISABLED) +def get_log_prefix(): + ''' + Return the current log prefix from the defaults for the formatter on the first logging handler. + ''' + try: + return next( + handler.formatter._style._defaults.get('prefix').rstrip().rstrip(':') + for handler in logging.getLogger().handlers + ) + except (StopIteration, AttributeError): + return None + + +def set_log_prefix(prefix): + ''' + Given a log prefix as a string, set it into the defaults for the formatters on all logging + handlers. + ''' + for handler in logging.getLogger().handlers: + try: + handler.formatter._style._defaults = {'prefix': f'{prefix}: ' if prefix else ''} + except AttributeError: + pass + + class Log_prefix: ''' A Python context manager for setting a log prefix so that it shows up in every subsequent @@ -215,23 +240,14 @@ class Log_prefix: up in every log message. But first, save off any original prefix so that it can be restored below. ''' - try: - self.original_prefix = next( - handler.formatter._style._defaults.get('prefix').rstrip().rstrip(':') - for handler in logging.getLogger().handlers - ) - except (StopIteration, AttributeError): - self.original_prefix = None - - for handler in logging.getLogger().handlers: - handler.formatter._style._defaults = {'prefix': f'{self.prefix}: ' if self.prefix else ''} + self.original_prefix = get_log_prefix() + set_log_prefix(self.prefix) def __exit__(self, exception, value, traceback): ''' Restore any original prefix. ''' - for handler in logging.getLogger().handlers: - handler.formatter._style._defaults = {'prefix': f'{self.original_prefix}: ' if self.original_prefix else ''} + set_log_prefix(self.original_prefix) def configure_logging( diff --git a/tests/end-to-end/test_override.py b/tests/end-to-end/test_override.py index a4fb08a7..ee8a64fe 100644 --- a/tests/end-to-end/test_override.py +++ b/tests/end-to-end/test_override.py @@ -28,7 +28,7 @@ def generate_configuration(config_path, repository_path): config_file.close() -def test_override_get_normalized(): +def test_override_gets_normalized(): temporary_directory = tempfile.mkdtemp() repository_path = os.path.join(temporary_directory, 'test.borg') diff --git a/tests/integration/hooks/monitoring/test_apprise.py b/tests/integration/hooks/monitoring/test_apprise.py index f1bcdf67..158e697b 100644 --- a/tests/integration/hooks/monitoring/test_apprise.py +++ b/tests/integration/hooks/monitoring/test_apprise.py @@ -14,7 +14,7 @@ def test_destroy_monitor_removes_apprise_handler(): ) ) - module.destroy_monitor(flexmock(), flexmock(), flexmock(), flexmock(), flexmock()) + module.destroy_monitor(flexmock(), flexmock(), flexmock(), flexmock()) assert logger.handlers == original_handlers @@ -23,6 +23,6 @@ def test_destroy_monitor_without_apprise_handler_does_not_raise(): logger = logging.getLogger() original_handlers = list(logger.handlers) - module.destroy_monitor(flexmock(), flexmock(), flexmock(), flexmock(), flexmock()) + module.destroy_monitor(flexmock(), flexmock(), flexmock(), flexmock()) assert logger.handlers == original_handlers diff --git a/tests/integration/hooks/monitoring/test_healthchecks.py b/tests/integration/hooks/monitoring/test_healthchecks.py index 9d6ceb4a..da9981b1 100644 --- a/tests/integration/hooks/monitoring/test_healthchecks.py +++ b/tests/integration/hooks/monitoring/test_healthchecks.py @@ -14,7 +14,7 @@ def test_destroy_monitor_removes_healthchecks_handler(): ) ) - module.destroy_monitor(flexmock(), flexmock(), flexmock(), flexmock(), flexmock()) + module.destroy_monitor(flexmock(), flexmock(), flexmock(), flexmock()) assert logger.handlers == original_handlers @@ -23,6 +23,6 @@ def test_destroy_monitor_without_healthchecks_handler_does_not_raise(): logger = logging.getLogger() original_handlers = list(logger.handlers) - module.destroy_monitor(flexmock(), flexmock(), flexmock(), flexmock(), flexmock()) + module.destroy_monitor(flexmock(), flexmock(), flexmock(), flexmock()) assert logger.handlers == original_handlers diff --git a/tests/integration/hooks/monitoring/test_loki.py b/tests/integration/hooks/monitoring/test_loki.py index e0f1cbba..a25a325f 100644 --- a/tests/integration/hooks/monitoring/test_loki.py +++ b/tests/integration/hooks/monitoring/test_loki.py @@ -65,8 +65,8 @@ def test_ping_monitor_adds_log_message(): assert any( map( lambda log: log - == f'{config_filename}: {module.MONITOR_STATE_TO_LOKI[module.monitor.State.FINISH]} backup', - map(lambda x: x[1], handler.buffer.root['streams'][0]['values']), + == f'{module.MONITOR_STATE_TO_LOKI[module.monitor.State.FINISH]} backup', + map(lambda value: value[1], handler.buffer.root['streams'][0]['values']), ) ) return @@ -82,7 +82,7 @@ def test_destroy_monitor_removes_log_handler(): config_filename = 'test.yaml' dry_run = True module.initialize_monitor(hook_config, flexmock(), config_filename, flexmock(), dry_run) - module.destroy_monitor(hook_config, flexmock(), config_filename, flexmock(), dry_run) + module.destroy_monitor(hook_config, flexmock(), flexmock(), dry_run) for handler in tuple(logging.getLogger().handlers): if isinstance(handler, module.Loki_log_handler): diff --git a/tests/unit/hooks/monitoring/test_apprise.py b/tests/unit/hooks/monitoring/test_apprise.py index 91118cb0..fd0e9615 100644 --- a/tests/unit/hooks/monitoring/test_apprise.py +++ b/tests/unit/hooks/monitoring/test_apprise.py @@ -379,7 +379,6 @@ def test_destroy_monitor_does_not_raise(): module.destroy_monitor( hook_config={}, config={}, - config_filename='test.yaml', monitoring_log_level=1, dry_run=False, ) From 71f1819f05ede67823cb3b60d20cf91e04798469 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Jan 2025 09:27:12 -0800 Subject: [PATCH 08/12] Some additional test coverage (#635). --- borgmatic/hooks/monitoring/cronhub.py | 8 +-- borgmatic/hooks/monitoring/cronitor.py | 8 +-- borgmatic/hooks/monitoring/healthchecks.py | 8 +-- borgmatic/hooks/monitoring/ntfy.py | 12 +--- borgmatic/hooks/monitoring/pagerduty.py | 4 +- borgmatic/hooks/monitoring/pushover.py | 4 +- borgmatic/hooks/monitoring/sentry.py | 8 +-- borgmatic/hooks/monitoring/uptime_kuma.py | 8 +-- borgmatic/hooks/monitoring/zabbix.py | 4 +- borgmatic/logger.py | 8 ++- tests/unit/config/test_paths.py | 4 +- .../unit/hooks/data_source/test_postgresql.py | 20 ++---- tests/unit/hooks/test_dispatch.py | 4 +- tests/unit/test_logger.py | 61 +++++++++++++++++++ 14 files changed, 89 insertions(+), 72 deletions(-) diff --git a/borgmatic/hooks/monitoring/cronhub.py b/borgmatic/hooks/monitoring/cronhub.py index dfc4892f..837d770b 100644 --- a/borgmatic/hooks/monitoring/cronhub.py +++ b/borgmatic/hooks/monitoring/cronhub.py @@ -28,9 +28,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev filename in any log entries. If this is a dry run, then don't actually ping anything. ''' if state not in MONITOR_STATE_TO_CRONHUB: - logger.debug( - f'Ignoring unsupported monitoring {state.name.lower()} in Cronhub hook' - ) + logger.debug(f'Ignoring unsupported monitoring {state.name.lower()} in Cronhub hook') return dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' @@ -54,9 +52,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev logger.warning(f'Cronhub error: {error}') -def destroy_monitor( - ping_url_or_uuid, config, monitoring_log_level, dry_run -): # pragma: no cover +def destroy_monitor(ping_url_or_uuid, config, monitoring_log_level, dry_run): # pragma: no cover ''' No destruction is necessary for this monitor. ''' diff --git a/borgmatic/hooks/monitoring/cronitor.py b/borgmatic/hooks/monitoring/cronitor.py index 994dab31..e8169312 100644 --- a/borgmatic/hooks/monitoring/cronitor.py +++ b/borgmatic/hooks/monitoring/cronitor.py @@ -28,9 +28,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev filename in any log entries. If this is a dry run, then don't actually ping anything. ''' if state not in MONITOR_STATE_TO_CRONITOR: - logger.debug( - f'Ignoring unsupported monitoring {state.name.lower()} in Cronitor hook' - ) + logger.debug(f'Ignoring unsupported monitoring {state.name.lower()} in Cronitor hook') return dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' @@ -49,9 +47,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev logger.warning(f'Cronitor error: {error}') -def destroy_monitor( - ping_url_or_uuid, config, monitoring_log_level, dry_run -): # pragma: no cover +def destroy_monitor(ping_url_or_uuid, config, monitoring_log_level, dry_run): # pragma: no cover ''' No destruction is necessary for this monitor. ''' diff --git a/borgmatic/hooks/monitoring/healthchecks.py b/borgmatic/hooks/monitoring/healthchecks.py index 1e145106..0b30d923 100644 --- a/borgmatic/hooks/monitoring/healthchecks.py +++ b/borgmatic/hooks/monitoring/healthchecks.py @@ -55,9 +55,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev dry_run_label = ' (dry run; not actually pinging)' if dry_run else '' if 'states' in hook_config and state.name.lower() not in hook_config['states']: - logger.info( - f'Skipping Healthchecks {state.name.lower()} ping due to configured states' - ) + logger.info(f'Skipping Healthchecks {state.name.lower()} ping due to configured states') return ping_url_is_uuid = re.search(r'\w{8}-\w{4}-\w{4}-\w{4}-\w{12}$', ping_url) @@ -68,9 +66,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev if hook_config.get('create_slug'): if ping_url_is_uuid: - logger.warning( - 'Healthchecks UUIDs do not support auto provisionning; ignoring' - ) + logger.warning('Healthchecks UUIDs do not support auto provisionning; ignoring') else: ping_url = f'{ping_url}?create=1' diff --git a/borgmatic/hooks/monitoring/ntfy.py b/borgmatic/hooks/monitoring/ntfy.py index 9fa28c80..f56a6047 100644 --- a/borgmatic/hooks/monitoring/ntfy.py +++ b/borgmatic/hooks/monitoring/ntfy.py @@ -62,13 +62,9 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev auth = requests.auth.HTTPBasicAuth(username, password) logger.info(f'Using basic auth with user {username} for ntfy') elif username is not None: - logger.warning( - 'Password missing for ntfy authentication, defaulting to no auth' - ) + logger.warning('Password missing for ntfy authentication, defaulting to no auth') elif password is not None: - logger.warning( - 'Username missing for ntfy authentication, defaulting to no auth' - ) + logger.warning('Username missing for ntfy authentication, defaulting to no auth') if not dry_run: logging.getLogger('urllib3').setLevel(logging.ERROR) @@ -80,9 +76,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev logger.warning(f'ntfy error: {error}') -def destroy_monitor( - ping_url_or_uuid, config, monitoring_log_level, dry_run -): # pragma: no cover +def destroy_monitor(ping_url_or_uuid, config, monitoring_log_level, dry_run): # pragma: no cover ''' No destruction is necessary for this monitor. ''' diff --git a/borgmatic/hooks/monitoring/pagerduty.py b/borgmatic/hooks/monitoring/pagerduty.py index 5e4aea4f..6afc27c2 100644 --- a/borgmatic/hooks/monitoring/pagerduty.py +++ b/borgmatic/hooks/monitoring/pagerduty.py @@ -72,9 +72,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev logger.warning(f'PagerDuty error: {error}') -def destroy_monitor( - ping_url_or_uuid, config, monitoring_log_level, dry_run -): # pragma: no cover +def destroy_monitor(ping_url_or_uuid, config, monitoring_log_level, dry_run): # pragma: no cover ''' No destruction is necessary for this monitor. ''' diff --git a/borgmatic/hooks/monitoring/pushover.py b/borgmatic/hooks/monitoring/pushover.py index 40f93d93..8211c32d 100644 --- a/borgmatic/hooks/monitoring/pushover.py +++ b/borgmatic/hooks/monitoring/pushover.py @@ -78,9 +78,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev logger.warning(f'Pushover error: {error}') -def destroy_monitor( - ping_url_or_uuid, config, monitoring_log_level, dry_run -): # pragma: no cover +def destroy_monitor(ping_url_or_uuid, config, monitoring_log_level, dry_run): # pragma: no cover ''' No destruction is necessary for this monitor. ''' diff --git a/borgmatic/hooks/monitoring/sentry.py b/borgmatic/hooks/monitoring/sentry.py index dbc8f014..857f85bc 100644 --- a/borgmatic/hooks/monitoring/sentry.py +++ b/borgmatic/hooks/monitoring/sentry.py @@ -38,9 +38,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev match = DATA_SOURCE_NAME_URL_PATTERN.match(data_source_name_url) if not match: - logger.warning( - f'Invalid Sentry data source name URL: {data_source_name_url}' - ) + logger.warning(f'Invalid Sentry data source name URL: {data_source_name_url}') return cron_url = f'{match.group("protocol")}://{match.group("hostname")}/api/{match.group("project_id")}/cron/{monitor_slug}/{match.group("username")}/' @@ -70,9 +68,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev logger.warning(f'Sentry error: {error}') -def destroy_monitor( - ping_url_or_uuid, config, monitoring_log_level, dry_run -): # pragma: no cover +def destroy_monitor(ping_url_or_uuid, config, monitoring_log_level, dry_run): # pragma: no cover ''' No destruction is necessary for this monitor. ''' diff --git a/borgmatic/hooks/monitoring/uptime_kuma.py b/borgmatic/hooks/monitoring/uptime_kuma.py index 9b06b750..015dde0b 100644 --- a/borgmatic/hooks/monitoring/uptime_kuma.py +++ b/borgmatic/hooks/monitoring/uptime_kuma.py @@ -28,9 +28,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev status = 'down' if state.name.lower() == 'fail' else 'up' push_url = hook_config.get('push_url', 'https://example.uptime.kuma/api/push/abcd1234') query = f'status={status}&msg={state.name.lower()}' - logger.info( - f'Pushing Uptime Kuma push_url {push_url}?{query} {dry_run_label}' - ) + logger.info(f'Pushing Uptime Kuma push_url {push_url}?{query} {dry_run_label}') logger.debug(f'Full Uptime Kuma state URL {push_url}?{query}') if dry_run: @@ -46,9 +44,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev logger.warning(f'Uptime Kuma error: {error}') -def destroy_monitor( - push_url_or_uuid, config, monitoring_log_level, dry_run -): # pragma: no cover +def destroy_monitor(push_url_or_uuid, config, monitoring_log_level, dry_run): # pragma: no cover ''' No destruction is necessary for this monitor. ''' diff --git a/borgmatic/hooks/monitoring/zabbix.py b/borgmatic/hooks/monitoring/zabbix.py index 17744a21..96b0a9da 100644 --- a/borgmatic/hooks/monitoring/zabbix.py +++ b/borgmatic/hooks/monitoring/zabbix.py @@ -126,9 +126,7 @@ def ping_monitor(hook_config, config, config_filename, state, monitoring_log_lev logger.warning(f'Zabbix error: {error}') -def destroy_monitor( - ping_url_or_uuid, config, monitoring_log_level, dry_run -): # pragma: no cover +def destroy_monitor(ping_url_or_uuid, config, monitoring_log_level, dry_run): # pragma: no cover ''' No destruction is necessary for this monitor. ''' diff --git a/borgmatic/logger.py b/borgmatic/logger.py index e6203fcd..4acaf50d 100644 --- a/borgmatic/logger.py +++ b/borgmatic/logger.py @@ -88,7 +88,7 @@ class Multi_stream_handler(logging.Handler): class Console_no_color_formatter(logging.Formatter): - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs): # pragma: no cover super(Console_no_color_formatter, self).__init__( '{prefix}{message}', style='{', defaults={'prefix': ''}, *args, **kwargs ) @@ -188,7 +188,8 @@ def add_custom_log_levels(): # pragma: no cover def get_log_prefix(): ''' - Return the current log prefix from the defaults for the formatter on the first logging handler. + Return the current log prefix from the defaults for the formatter on the first logging handler, + set by set_log_prefix(). Return None if no such prefix exists. ''' try: return next( @@ -219,13 +220,14 @@ class Log_prefix: Example use as a context manager: - + with borgmatic.logger.Log_prefix('myprefix'): do_something_that_logs() For the scope of that "with" statement, any logs created are prefixed with "myprefix: ". Afterwards, the prefix gets restored to whatever it was prior to the context manager. ''' + def __init__(self, prefix): ''' Given the desired log prefix, save it for use below. Set prefix to None to disable any diff --git a/tests/unit/config/test_paths.py b/tests/unit/config/test_paths.py index 12975b9d..65e3e477 100644 --- a/tests/unit/config/test_paths.py +++ b/tests/unit/config/test_paths.py @@ -84,9 +84,7 @@ def test_runtime_directory_with_relative_config_option_errors(): config = {'user_runtime_directory': 'run', 'borgmatic_source_directory': '/nope'} with pytest.raises(ValueError): - with module.Runtime_directory( - config - ) as borgmatic_runtime_directory: # noqa: F841 + with module.Runtime_directory(config) as borgmatic_runtime_directory: # noqa: F841 pass diff --git a/tests/unit/hooks/data_source/test_postgresql.py b/tests/unit/hooks/data_source/test_postgresql.py index 4188f96d..845859f7 100644 --- a/tests/unit/hooks/data_source/test_postgresql.py +++ b/tests/unit/hooks/data_source/test_postgresql.py @@ -61,25 +61,19 @@ def test_make_extra_environment_without_ssl_mode_does_not_set_ssl_mode(): def test_database_names_to_dump_passes_through_individual_database_name(): database = {'name': 'foo'} - assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ( - 'foo', - ) + assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ('foo',) def test_database_names_to_dump_passes_through_individual_database_name_with_format(): database = {'name': 'foo', 'format': 'custom'} - assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ( - 'foo', - ) + assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ('foo',) def test_database_names_to_dump_passes_through_all_without_format(): database = {'name': 'all'} - assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ( - 'all', - ) + assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ('all',) def test_database_names_to_dump_with_all_and_format_and_dry_run_bails(): @@ -166,9 +160,7 @@ def test_database_names_to_dump_with_all_and_format_excludes_particular_database 'foo,test,\ntemplate0,test,blah' ) - assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ( - 'foo', - ) + assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ('foo',) def test_database_names_to_dump_with_all_and_psql_command_uses_custom_command(): @@ -194,9 +186,7 @@ def test_database_names_to_dump_with_all_and_psql_command_uses_custom_command(): extra_environment=object, ).and_return('foo,text').once() - assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ( - 'foo', - ) + assert module.database_names_to_dump(database, flexmock(), dry_run=False) == ('foo',) def test_use_streaming_true_for_any_non_directory_format_databases(): diff --git a/tests/unit/hooks/test_dispatch.py b/tests/unit/hooks/test_dispatch.py index d0eca83d..80f8c046 100644 --- a/tests/unit/hooks/test_dispatch.py +++ b/tests/unit/hooks/test_dispatch.py @@ -74,9 +74,7 @@ def test_call_hook_strips_databases_suffix_from_hook_name(): config['super_hook_databases'], config, 55, value=66 ).and_return(expected_return_value).once() - return_value = module.call_hook( - 'hook_function', config, 'super_hook_databases', 55, value=66 - ) + return_value = module.call_hook('hook_function', config, 'super_hook_databases', 55, value=66) assert return_value == expected_return_value diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index 97aeb63a..5a4d60c9 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -228,6 +228,67 @@ def test_add_logging_level_skips_global_setting_if_already_set(): module.add_logging_level('PLAID', 99) +def test_get_log_prefix_gets_prefix_from_first_handler(): + flexmock(module.logging).should_receive('getLogger').and_return( + flexmock( + handlers=[ + flexmock( + formatter=flexmock( + _style=flexmock(_defaults=flexmock(get=lambda name: 'myprefix: ')) + ) + ), + flexmock(), + ], + removeHandler=lambda handler: None, + ) + ) + + assert module.get_log_prefix() == 'myprefix' + + +def test_get_log_prefix_with_no_handlers_does_not_raise(): + flexmock(module.logging).should_receive('getLogger').and_return( + flexmock( + handlers=[], + removeHandler=lambda handler: None, + ) + ) + + assert module.get_log_prefix() == None + + +def test_get_log_prefix_with_no_formatters_does_not_raise(): + flexmock(module.logging).should_receive('getLogger').and_return( + flexmock( + handlers=[ + flexmock(), + flexmock(), + ], + removeHandler=lambda handler: None, + ) + ) + + assert module.get_log_prefix() == None + + +def test_get_log_prefix_with_no_prefix_does_not_raise(): + flexmock(module.logging).should_receive('getLogger').and_return( + flexmock( + handlers=[ + flexmock( + formatter=flexmock( + _style=flexmock(_defaults=flexmock(get=lambda name: None)) + ) + ), + flexmock(), + ], + removeHandler=lambda handler: None, + ) + ) + + assert module.get_log_prefix() == None + + def test_configure_logging_with_syslog_log_level_probes_for_log_socket_on_linux(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.ANSWER From 7e344e6e0adce4d8e31420363aa08e6e8d3d41fb Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Jan 2025 12:25:28 -0800 Subject: [PATCH 09/12] Complete test coverage for new code (#635). --- borgmatic/logger.py | 2 +- tests/unit/test_logger.py | 77 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 75 insertions(+), 4 deletions(-) diff --git a/borgmatic/logger.py b/borgmatic/logger.py index 4acaf50d..a81541d1 100644 --- a/borgmatic/logger.py +++ b/borgmatic/logger.py @@ -203,7 +203,7 @@ def get_log_prefix(): def set_log_prefix(prefix): ''' Given a log prefix as a string, set it into the defaults for the formatters on all logging - handlers. + handlers. Note that this overwrites any existing defaults. ''' for handler in logging.getLogger().handlers: try: diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index 5a4d60c9..6d8b0049 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -276,9 +276,7 @@ def test_get_log_prefix_with_no_prefix_does_not_raise(): flexmock( handlers=[ flexmock( - formatter=flexmock( - _style=flexmock(_defaults=flexmock(get=lambda name: None)) - ) + formatter=flexmock(_style=flexmock(_defaults=flexmock(get=lambda name: None))) ), flexmock(), ], @@ -289,6 +287,79 @@ def test_get_log_prefix_with_no_prefix_does_not_raise(): assert module.get_log_prefix() == None +def test_set_log_prefix_updates_all_handlers(): + styles = ( + flexmock(_defaults=None), + flexmock(_defaults=None), + ) + + flexmock(module.logging).should_receive('getLogger').and_return( + flexmock( + handlers=[ + flexmock( + formatter=flexmock( + _style=styles[0], + ) + ), + flexmock( + formatter=flexmock( + _style=styles[1], + ) + ), + ], + removeHandler=lambda handler: None, + ) + ) + + module.set_log_prefix('myprefix') + + for style in styles: + assert style._defaults == {'prefix': 'myprefix: '} + + +def test_set_log_prefix_skips_handlers_without_a_formatter(): + style = flexmock(_defaults=None) + + flexmock(module.logging).should_receive('getLogger').and_return( + flexmock( + handlers=[ + flexmock( + formatter=None, + ), + flexmock(), + flexmock( + formatter=flexmock( + _style=style, + ) + ), + ], + removeHandler=lambda handler: None, + ) + ) + + module.set_log_prefix('myprefix') + + assert style._defaults == {'prefix': 'myprefix: '} + + +def test_log_prefix_sets_prefix_and_then_restores_no_prefix_after(): + flexmock(module).should_receive('get_log_prefix').and_return(None) + flexmock(module).should_receive('set_log_prefix').with_args('myprefix').once() + flexmock(module).should_receive('set_log_prefix').with_args(None).once() + + with module.Log_prefix('myprefix'): + pass + + +def test_log_prefix_sets_prefix_and_then_restores_original_prefix_after(): + flexmock(module).should_receive('get_log_prefix').and_return('original') + flexmock(module).should_receive('set_log_prefix').with_args('myprefix').once() + flexmock(module).should_receive('set_log_prefix').with_args('original').once() + + with module.Log_prefix('myprefix'): + pass + + def test_configure_logging_with_syslog_log_level_probes_for_log_socket_on_linux(): flexmock(module).should_receive('add_custom_log_levels') flexmock(module.logging).ANSWER = module.ANSWER From 63c39be55f7ed87464852373c9a20cd84a3de596 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Mon, 27 Jan 2025 12:28:36 -0800 Subject: [PATCH 10/12] Fix flaking issues (#635). --- borgmatic/actions/check.py | 2 +- borgmatic/commands/borgmatic.py | 2 +- borgmatic/hooks/data_source/mariadb.py | 2 +- borgmatic/hooks/data_source/mysql.py | 2 +- borgmatic/hooks/data_source/postgresql.py | 2 +- tests/unit/test_logger.py | 6 +++--- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index 2d514ad0..be2d0c61 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -709,7 +709,7 @@ def run_check( **hook_context, ) - logger.info(f'Running consistency checks') + logger.info('Running consistency checks') repository_id = borgmatic.borg.check.get_repository_id( repository['path'], diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 1bcdbb7f..76f16a43 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -176,7 +176,7 @@ def run_configuration(config_filename, config, config_paths, arguments): continue yield from log_error_records( - f'Error running actions for repository', + 'Error running actions for repository', error, ) encountered_error = error diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index abf7ba96..d309d9e5 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -49,7 +49,7 @@ def database_names_to_dump(database, extra_environment, dry_run): + ('--skip-column-names', '--batch') + ('--execute', 'show schemas') ) - logger.debug(f'Querying for "all" MariaDB databases to dump') + logger.debug('Querying for "all" MariaDB databases to dump') show_output = execute_command_and_capture_output( show_command, extra_environment=extra_environment ) diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index d302d1f5..a477d2ac 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -49,7 +49,7 @@ def database_names_to_dump(database, extra_environment, dry_run): + ('--skip-column-names', '--batch') + ('--execute', 'show schemas') ) - logger.debug(f'Querying for "all" MySQL databases to dump') + logger.debug('Querying for "all" MySQL databases to dump') show_output = execute_command_and_capture_output( show_command, extra_environment=extra_environment ) diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index 94a6e1f5..40c3e57b 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -85,7 +85,7 @@ def database_names_to_dump(database, extra_environment, dry_run): + (('--username', database['username']) if 'username' in database else ()) + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ()) ) - logger.debug(f'Querying for "all" PostgreSQL databases to dump') + logger.debug('Querying for "all" PostgreSQL databases to dump') list_output = execute_command_and_capture_output( list_command, extra_environment=extra_environment ) diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index 6d8b0049..ef694413 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -254,7 +254,7 @@ def test_get_log_prefix_with_no_handlers_does_not_raise(): ) ) - assert module.get_log_prefix() == None + assert module.get_log_prefix() is None def test_get_log_prefix_with_no_formatters_does_not_raise(): @@ -268,7 +268,7 @@ def test_get_log_prefix_with_no_formatters_does_not_raise(): ) ) - assert module.get_log_prefix() == None + assert module.get_log_prefix() is None def test_get_log_prefix_with_no_prefix_does_not_raise(): @@ -284,7 +284,7 @@ def test_get_log_prefix_with_no_prefix_does_not_raise(): ) ) - assert module.get_log_prefix() == None + assert module.get_log_prefix() is None def test_set_log_prefix_updates_all_handlers(): From 5a9066940f49977858abf8f411db1b48d1df4630 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 28 Jan 2025 23:06:22 -0800 Subject: [PATCH 11/12] Add monitoring end-to-end tests (#635). --- docs/how-to/monitor-your-backups.md | 16 +- tests/end-to-end/hooks/__init__.py | 0 .../end-to-end/hooks/data_source/__init__.py | 0 .../{ => hooks/data_source}/test_btrfs.py | 0 .../{ => hooks/data_source}/test_database.py | 0 .../{ => hooks/data_source}/test_lvm.py | 0 .../{ => hooks/data_source}/test_zfs.py | 0 .../hooks/monitoring/test_monitoring.py | 141 ++++++++++++++++++ 8 files changed, 147 insertions(+), 10 deletions(-) create mode 100644 tests/end-to-end/hooks/__init__.py create mode 100644 tests/end-to-end/hooks/data_source/__init__.py rename tests/end-to-end/{ => hooks/data_source}/test_btrfs.py (100%) rename tests/end-to-end/{ => hooks/data_source}/test_database.py (100%) rename tests/end-to-end/{ => hooks/data_source}/test_lvm.py (100%) rename tests/end-to-end/{ => hooks/data_source}/test_zfs.py (100%) create mode 100644 tests/end-to-end/hooks/monitoring/test_monitoring.py diff --git a/docs/how-to/monitor-your-backups.md b/docs/how-to/monitor-your-backups.md index efe34423..43acdb08 100644 --- a/docs/how-to/monitor-your-backups.md +++ b/docs/how-to/monitor-your-backups.md @@ -478,22 +478,18 @@ your Loki API push URL. Here's an example: ```yaml loki: url: http://localhost:3100/loki/api/v1/push + + labels: + app: borgmatic + hostname: example.org ``` With this configuration, borgmatic sends its logs to your Loki instance as any of the `create`, `prune`, `compact`, or `check` actions are run. Then, after the actions complete, borgmatic notifies Loki of success or failure. -This hook supports sending arbitrary labels to Loki. For instance: - -```yaml -loki: - url: http://localhost:3100/loki/api/v1/push - - labels: - app: borgmatic - hostname: example.org -``` +This hook supports sending arbitrary labels to Loki. At least one label is +required. There are also a few placeholders you can optionally use as label values: diff --git a/tests/end-to-end/hooks/__init__.py b/tests/end-to-end/hooks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/end-to-end/hooks/data_source/__init__.py b/tests/end-to-end/hooks/data_source/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/end-to-end/test_btrfs.py b/tests/end-to-end/hooks/data_source/test_btrfs.py similarity index 100% rename from tests/end-to-end/test_btrfs.py rename to tests/end-to-end/hooks/data_source/test_btrfs.py diff --git a/tests/end-to-end/test_database.py b/tests/end-to-end/hooks/data_source/test_database.py similarity index 100% rename from tests/end-to-end/test_database.py rename to tests/end-to-end/hooks/data_source/test_database.py diff --git a/tests/end-to-end/test_lvm.py b/tests/end-to-end/hooks/data_source/test_lvm.py similarity index 100% rename from tests/end-to-end/test_lvm.py rename to tests/end-to-end/hooks/data_source/test_lvm.py diff --git a/tests/end-to-end/test_zfs.py b/tests/end-to-end/hooks/data_source/test_zfs.py similarity index 100% rename from tests/end-to-end/test_zfs.py rename to tests/end-to-end/hooks/data_source/test_zfs.py diff --git a/tests/end-to-end/hooks/monitoring/test_monitoring.py b/tests/end-to-end/hooks/monitoring/test_monitoring.py new file mode 100644 index 00000000..4e8e61da --- /dev/null +++ b/tests/end-to-end/hooks/monitoring/test_monitoring.py @@ -0,0 +1,141 @@ +import http.server +import json +import os +import shutil +import subprocess +import sys +import tempfile +import threading + +import pytest + + +def generate_configuration(config_path, repository_path, monitoring_hook_configuration): + ''' + Generate borgmatic configuration into a file at the config path, and update the defaults so as + to work for testing, including updating the source directories, injecting the given repository + path, and tacking on an encryption passphrase. + ''' + subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' ')) + config = ( + open(config_path) + .read() + .replace('ssh://user@backupserver/./sourcehostname.borg', repository_path) + .replace('- path: /mnt/backup', '') + .replace('label: local', '') + .replace('- /home/user/path with spaces', '') + .replace('- /home', f'- {config_path}') + .replace('- /etc', '') + .replace('- /var/log/syslog*', '') + + '\nencryption_passphrase: "test"' + + f'\n{monitoring_hook_configuration}' + ) + config_file = open(config_path, 'w') + config_file.write(config) + config_file.close() + + +class Web_server(http.server.BaseHTTPRequestHandler): + def handle_method(self): + self.send_response(http.HTTPStatus.OK) + self.send_header('Content-type', 'text/html') + self.end_headers() + self.wfile.write(''.encode('utf-8')) + + def do_GET(self): + self.handle_method() + + def do_POST(self): + self.handle_method() + + +def serve_web_request(count): + for index in range(0, count): + with http.server.HTTPServer(('localhost', 12345), Web_server) as server: + server.handle_request() + + +class Background_web_server: + def __init__(self, expected_request_count): + self.expected_request_count = expected_request_count + + def __enter__(self): + self.thread = threading.Thread(target=lambda: serve_web_request(count=self.expected_request_count)) + self.thread.start() + + def __exit__(self, exception, value, traceback): + self.thread.join() + + +START_AND_FINISH = 2 +START_LOG_AND_FINISH = 3 + + +@pytest.mark.parametrize( + 'monitoring_hook_configuration,expected_request_count', + ( + ( + 'cronhub:\n ping_url: http://localhost:12345/start/1f5e3410-254c-11e8-b61d-55875966d031', + START_AND_FINISH, + ), + ( + 'cronitor:\n ping_url: http://localhost:12345/d3x0c1', + START_AND_FINISH, + ), + ( + 'healthchecks:\n ping_url: http://localhost:12345/addffa72-da17-40ae-be9c-ff591afb942a', + START_LOG_AND_FINISH, + ), + ( + 'loki:\n url: http://localhost:12345/loki/api/v1/push\n labels:\n app: borgmatic', + START_AND_FINISH, + ), + ( + 'ntfy:\n topic: my-unique-topic\n server: http://localhost:12345\n states: [start, finish]', + START_AND_FINISH, + ), + ( + 'sentry:\n data_source_name_url: http://5f80ec@localhost:12345/203069\n monitor_slug: mymonitor', + START_AND_FINISH, + ), + ( + 'uptime_kuma:\n push_url: http://localhost:12345/api/push/abcd1234', + START_AND_FINISH, + ), + ( + 'zabbix:\n itemid: 1\n server: http://localhost:12345/zabbix/api_jsonrpc.php\n api_key: mykey\n states: [start, finish]', + START_AND_FINISH, + ), + ), +) +def test_borgmatic_command(monitoring_hook_configuration, expected_request_count): + # Create a Borg repository. + temporary_directory = tempfile.mkdtemp() + repository_path = os.path.join(temporary_directory, 'test.borg') + extract_path = os.path.join(temporary_directory, 'extract') + + original_working_directory = os.getcwd() + os.mkdir(extract_path) + os.chdir(extract_path) + + try: + config_path = os.path.join(temporary_directory, 'test.yaml') + generate_configuration(config_path, repository_path, monitoring_hook_configuration) + + subprocess.check_call( + f'borgmatic -v 2 --config {config_path} repo-create --encryption repokey'.split(' ') + ) + + with Background_web_server(expected_request_count): + # Run borgmatic to generate a backup archive, and then list it to make sure it exists. + subprocess.check_call(f'borgmatic -v 2 --config {config_path}'.split(' ')) + output = subprocess.check_output( + f'borgmatic --config {config_path} list --json'.split(' ') + ).decode(sys.stdout.encoding) + parsed_output = json.loads(output) + + assert len(parsed_output) == 1 + assert len(parsed_output[0]['archives']) == 1 + finally: + os.chdir(original_working_directory) + shutil.rmtree(temporary_directory) From 67c22e464a9db21e91740df52539b7bb4c183c0b Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Wed, 29 Jan 2025 08:00:42 -0800 Subject: [PATCH 12/12] Code formatting (#635). --- tests/end-to-end/hooks/monitoring/test_monitoring.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/end-to-end/hooks/monitoring/test_monitoring.py b/tests/end-to-end/hooks/monitoring/test_monitoring.py index 4e8e61da..5aa264e8 100644 --- a/tests/end-to-end/hooks/monitoring/test_monitoring.py +++ b/tests/end-to-end/hooks/monitoring/test_monitoring.py @@ -60,7 +60,9 @@ class Background_web_server: self.expected_request_count = expected_request_count def __enter__(self): - self.thread = threading.Thread(target=lambda: serve_web_request(count=self.expected_request_count)) + self.thread = threading.Thread( + target=lambda: serve_web_request(count=self.expected_request_count) + ) self.thread.start() def __exit__(self, exception, value, traceback): @@ -106,7 +108,7 @@ START_LOG_AND_FINISH = 3 'zabbix:\n itemid: 1\n server: http://localhost:12345/zabbix/api_jsonrpc.php\n api_key: mykey\n states: [start, finish]', START_AND_FINISH, ), - ), + ), ) def test_borgmatic_command(monitoring_hook_configuration, expected_request_count): # Create a Borg repository.