From 310b287cff73c432c21853bb31011a8755dde86c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 6 Jan 2026 15:39:19 -0800 Subject: [PATCH] Make execute_command_with_processes() a generator like execute_command_and_capture_output() (#1225). --- borgmatic/actions/check.py | 4 +-- borgmatic/borg/create.py | 20 ++++++------ borgmatic/execute.py | 38 +++++++++++------------ borgmatic/hooks/data_source/mariadb.py | 16 +++++----- borgmatic/hooks/data_source/mongodb.py | 14 +++++---- borgmatic/hooks/data_source/mysql.py | 16 +++++----- borgmatic/hooks/data_source/postgresql.py | 16 +++++----- borgmatic/hooks/data_source/sqlite.py | 14 +++++---- 8 files changed, 73 insertions(+), 65 deletions(-) diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index 166ca83c..187a7cfa 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -529,9 +529,7 @@ def compare_spot_check_hashes( for part in shlex.split(spot_check_config.get('xxh64sum_command', 'xxh64sum')) ) + tuple( - path - for path in source_sample_paths_subset - if path in hashable_source_sample_path + path for path in source_sample_paths_subset if path in hashable_source_sample_path ), working_directory=working_directory, ) diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 52cf0338..4bf958b5 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -383,15 +383,17 @@ def create_archive( borg_exit_codes = config.get('borg_exit_codes') if stream_processes: - return execute_command_with_processes( - create_flags + create_positional_arguments, - stream_processes, - output_log_level, - output_file, - working_directory=working_directory, - environment=environment.make_environment(config), - borg_local_path=local_path, - borg_exit_codes=borg_exit_codes, + return '\n'.join( + execute_command_with_processes( + create_flags + create_positional_arguments, + stream_processes, + output_log_level, + output_file, + working_directory=working_directory, + environment=environment.make_environment(config), + borg_local_path=local_path, + borg_exit_codes=borg_exit_codes, + ) ) if output_log_level is None: diff --git a/borgmatic/execute.py b/borgmatic/execute.py index bcc54cad..f36c7aaf 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -440,7 +440,7 @@ def execute_command_and_capture_output( ): ''' Execute the given command (a sequence of command/argument strings), capturing and returning its - output (stdout) as a generator that yields a line at a time. The generator must be consumed in + output (stdout) as a generator that yields one line at a time. The generator must be consumed in order for the called command to execute. If an input file descriptor is given, then pipe it to the command's stdin. If capture stderr is @@ -509,13 +509,16 @@ def execute_command_with_processes( If an open output file object is given, then write stdout to the file and only log stderr. But if output log level is None, instead suppress logging and return the captured output for (only) - the given command. If an open input file object is given, then read stdin from the file. If - shell is True, execute the command within a shell. If an environment variables dict is given, - then pass it into the command. If a working directory is given, use that as the present working - directory when running the command. If a Borg local path is given, then for any matching command - or process (regardless of arguments), treat exit code 1 as a warning instead of an error. But if - Borg exit codes are given as a sequence of exit code configuration dicts, then use that - configuration to decide what's an error and what's a warning. + the given command as a generator that yields one line at a time. The generator must be consumed + in order for the called command to execute—regardless of the output log level. + + If an open input file object is given, then read stdin from the file. If shell is True, execute + the command within a shell. If an environment variables dict is given, then pass it into the + command. If a working directory is given, use that as the present working directory when running + the command. If a Borg local path is given, then for any matching command or process (regardless + of arguments), treat exit code 1 as a warning instead of an error. But if Borg exit codes are + given as a sequence of exit code configuration dicts, then use that configuration to decide + what's an error and what's a warning. Raise subprocesses.CalledProcessError if an error occurs while running the command or in the upstream process. @@ -546,17 +549,12 @@ def execute_command_with_processes( raise with borgmatic.logger.Log_prefix(None): # Log command output without any prefix. - captured_lines = tuple( - log_outputs( - (*processes, command_process), - (input_file, output_file), - output_log_level, - borg_local_path, - borg_exit_codes, - ) + captured_lines = log_outputs( + (*processes, command_process), + (input_file, output_file), + output_log_level, + borg_local_path, + borg_exit_codes, ) - if output_log_level is None: - return '\n'.join(captured_lines) - - return None + return captured_lines diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index da324da6..84a0fd71 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -516,11 +516,13 @@ def restore_data_source_dump( # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning # if the restore paths don't exist in the archive. - execute_command_with_processes( - restore_command, - [extract_process], - output_log_level=logging.DEBUG, - input_file=extract_process.stdout, - environment=environment, - working_directory=borgmatic.config.paths.get_working_directory(config), + tuple( + execute_command_with_processes( + restore_command, + [extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + environment=environment, + working_directory=borgmatic.config.paths.get_working_directory(config), + ) ) diff --git a/borgmatic/hooks/data_source/mongodb.py b/borgmatic/hooks/data_source/mongodb.py index 63311ac4..0ef03132 100644 --- a/borgmatic/hooks/data_source/mongodb.py +++ b/borgmatic/hooks/data_source/mongodb.py @@ -289,12 +289,14 @@ def restore_data_source_dump( # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning # if the restore paths don't exist in the archive. - execute_command_with_processes( - restore_command, - [extract_process] if extract_process else [], - output_log_level=logging.DEBUG, - input_file=extract_process.stdout if extract_process else None, - working_directory=borgmatic.config.paths.get_working_directory(config), + tuple( + execute_command_with_processes( + restore_command, + [extract_process] if extract_process else [], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout if extract_process else None, + working_directory=borgmatic.config.paths.get_working_directory(config), + ) ) diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index 3198a28c..9bed3683 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -453,11 +453,13 @@ def restore_data_source_dump( # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning # if the restore paths don't exist in the archive. - execute_command_with_processes( - restore_command, - [extract_process], - output_log_level=logging.DEBUG, - input_file=extract_process.stdout, - environment=environment, - working_directory=borgmatic.config.paths.get_working_directory(config), + tuple( + execute_command_with_processes( + restore_command, + [extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + environment=environment, + working_directory=borgmatic.config.paths.get_working_directory(config), + ) ) diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index 675913f0..b8397d0e 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -434,13 +434,15 @@ def restore_data_source_dump( # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning # if the restore paths don't exist in the archive. - execute_command_with_processes( - restore_command, - [extract_process] if extract_process else [], - output_log_level=logging.DEBUG, - input_file=extract_process.stdout if extract_process else None, - environment=environment, - working_directory=borgmatic.config.paths.get_working_directory(config), + tuple( + execute_command_with_processes( + restore_command, + [extract_process] if extract_process else [], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout if extract_process else None, + environment=environment, + working_directory=borgmatic.config.paths.get_working_directory(config), + ) ) execute_command( analyze_command, diff --git a/borgmatic/hooks/data_source/sqlite.py b/borgmatic/hooks/data_source/sqlite.py index 43c9bd07..fcdcfcdb 100644 --- a/borgmatic/hooks/data_source/sqlite.py +++ b/borgmatic/hooks/data_source/sqlite.py @@ -217,10 +217,12 @@ def restore_data_source_dump( restore_command = (*sqlite_restore_command, '-bail', shlex.quote(database_path)) # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning # if the restore paths don't exist in the archive. - execute_command_with_processes( - restore_command, - [extract_process], - output_log_level=logging.DEBUG, - input_file=extract_process.stdout, - working_directory=borgmatic.config.paths.get_working_directory(config), + tuple( + execute_command_with_processes( + restore_command, + [extract_process], + output_log_level=logging.DEBUG, + input_file=extract_process.stdout, + working_directory=borgmatic.config.paths.get_working_directory(config), + ) )