Make execute_command_with_processes() a generator like execute_command_and_capture_output() (#1225).

borgmatic/actions/check.py

@@ -529,9 +529,7 @@ def compare_spot_check_hashes(
                 for part in shlex.split(spot_check_config.get('xxh64sum_command', 'xxh64sum'))
             )
             + tuple(
-                path
-                for path in source_sample_paths_subset
-                if path in hashable_source_sample_path
+                path for path in source_sample_paths_subset if path in hashable_source_sample_path
             ),
             working_directory=working_directory,
         )

borgmatic/borg/create.py

@@ -383,15 +383,17 @@ def create_archive(
     borg_exit_codes = config.get('borg_exit_codes')
 
     if stream_processes:
-        return execute_command_with_processes(
-            create_flags + create_positional_arguments,
-            stream_processes,
-            output_log_level,
-            output_file,
-            working_directory=working_directory,
-            environment=environment.make_environment(config),
-            borg_local_path=local_path,
-            borg_exit_codes=borg_exit_codes,
+        return '\n'.join(
+            execute_command_with_processes(
+                create_flags + create_positional_arguments,
+                stream_processes,
+                output_log_level,
+                output_file,
+                working_directory=working_directory,
+                environment=environment.make_environment(config),
+                borg_local_path=local_path,
+                borg_exit_codes=borg_exit_codes,
+            )
         )
 
     if output_log_level is None:

borgmatic/execute.py

@@ -440,7 +440,7 @@ def execute_command_and_capture_output(
 ):
     '''
     Execute the given command (a sequence of command/argument strings), capturing and returning its
-    output (stdout) as a generator that yields a line at a time. The generator must be consumed in
+    output (stdout) as a generator that yields one line at a time. The generator must be consumed in
     order for the called command to execute.
 
     If an input file descriptor is given, then pipe it to the command's stdin. If capture stderr is
@@ -509,13 +509,16 @@ def execute_command_with_processes(
 
     If an open output file object is given, then write stdout to the file and only log stderr. But
     if output log level is None, instead suppress logging and return the captured output for (only)
-    the given command. If an open input file object is given, then read stdin from the file. If
-    shell is True, execute the command within a shell. If an environment variables dict is given,
-    then pass it into the command. If a working directory is given, use that as the present working
-    directory when running the command. If a Borg local path is given, then for any matching command
-    or process (regardless of arguments), treat exit code 1 as a warning instead of an error. But if
-    Borg exit codes are given as a sequence of exit code configuration dicts, then use that
-    configuration to decide what's an error and what's a warning.
+    the given command as a generator that yields one line at a time. The generator must be consumed
+    in order for the called command to execute—regardless of the output log level.
+
+    If an open input file object is given, then read stdin from the file. If shell is True, execute
+    the command within a shell. If an environment variables dict is given, then pass it into the
+    command. If a working directory is given, use that as the present working directory when running
+    the command. If a Borg local path is given, then for any matching command or process (regardless
+    of arguments), treat exit code 1 as a warning instead of an error. But if Borg exit codes are
+    given as a sequence of exit code configuration dicts, then use that configuration to decide
+    what's an error and what's a warning.
 
     Raise subprocesses.CalledProcessError if an error occurs while running the command or in the
     upstream process.
@@ -546,17 +549,12 @@ def execute_command_with_processes(
         raise
 
     with borgmatic.logger.Log_prefix(None):  # Log command output without any prefix.
-        captured_lines = tuple(
-            log_outputs(
-                (*processes, command_process),
-                (input_file, output_file),
-                output_log_level,
-                borg_local_path,
-                borg_exit_codes,
-            )
+        captured_lines = log_outputs(
+            (*processes, command_process),
+            (input_file, output_file),
+            output_log_level,
+            borg_local_path,
+            borg_exit_codes,
         )
 
-    if output_log_level is None:
-        return '\n'.join(captured_lines)
-
-    return None
+    return captured_lines

borgmatic/hooks/data_source/mariadb.py

@@ -516,11 +516,13 @@ def restore_data_source_dump(
 
     # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
     # if the restore paths don't exist in the archive.
-    execute_command_with_processes(
-        restore_command,
-        [extract_process],
-        output_log_level=logging.DEBUG,
-        input_file=extract_process.stdout,
-        environment=environment,
-        working_directory=borgmatic.config.paths.get_working_directory(config),
+    tuple(
+        execute_command_with_processes(
+            restore_command,
+            [extract_process],
+            output_log_level=logging.DEBUG,
+            input_file=extract_process.stdout,
+            environment=environment,
+            working_directory=borgmatic.config.paths.get_working_directory(config),
+        )
     )

borgmatic/hooks/data_source/mongodb.py

@@ -289,12 +289,14 @@ def restore_data_source_dump(
 
     # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
     # if the restore paths don't exist in the archive.
-    execute_command_with_processes(
-        restore_command,
-        [extract_process] if extract_process else [],
-        output_log_level=logging.DEBUG,
-        input_file=extract_process.stdout if extract_process else None,
-        working_directory=borgmatic.config.paths.get_working_directory(config),
+    tuple(
+        execute_command_with_processes(
+            restore_command,
+            [extract_process] if extract_process else [],
+            output_log_level=logging.DEBUG,
+            input_file=extract_process.stdout if extract_process else None,
+            working_directory=borgmatic.config.paths.get_working_directory(config),
+        )
     )
 
 

borgmatic/hooks/data_source/mysql.py

@@ -453,11 +453,13 @@ def restore_data_source_dump(
 
     # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
     # if the restore paths don't exist in the archive.
-    execute_command_with_processes(
-        restore_command,
-        [extract_process],
-        output_log_level=logging.DEBUG,
-        input_file=extract_process.stdout,
-        environment=environment,
-        working_directory=borgmatic.config.paths.get_working_directory(config),
+    tuple(
+        execute_command_with_processes(
+            restore_command,
+            [extract_process],
+            output_log_level=logging.DEBUG,
+            input_file=extract_process.stdout,
+            environment=environment,
+            working_directory=borgmatic.config.paths.get_working_directory(config),
+        )
     )

borgmatic/hooks/data_source/postgresql.py

@@ -434,13 +434,15 @@ def restore_data_source_dump(
 
     # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
     # if the restore paths don't exist in the archive.
-    execute_command_with_processes(
-        restore_command,
-        [extract_process] if extract_process else [],
-        output_log_level=logging.DEBUG,
-        input_file=extract_process.stdout if extract_process else None,
-        environment=environment,
-        working_directory=borgmatic.config.paths.get_working_directory(config),
+    tuple(
+        execute_command_with_processes(
+            restore_command,
+            [extract_process] if extract_process else [],
+            output_log_level=logging.DEBUG,
+            input_file=extract_process.stdout if extract_process else None,
+            environment=environment,
+            working_directory=borgmatic.config.paths.get_working_directory(config),
+        )
     )
     execute_command(
         analyze_command,

borgmatic/hooks/data_source/sqlite.py

@@ -217,10 +217,12 @@ def restore_data_source_dump(
     restore_command = (*sqlite_restore_command, '-bail', shlex.quote(database_path))
     # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
     # if the restore paths don't exist in the archive.
-    execute_command_with_processes(
-        restore_command,
-        [extract_process],
-        output_log_level=logging.DEBUG,
-        input_file=extract_process.stdout,
-        working_directory=borgmatic.config.paths.get_working_directory(config),
+    tuple(
+        execute_command_with_processes(
+            restore_command,
+            [extract_process],
+            output_log_level=logging.DEBUG,
+            input_file=extract_process.stdout,
+            working_directory=borgmatic.config.paths.get_working_directory(config),
+        )
     )