Reduce memory consumption of "create" action path validation (#1225)

borgmatic/actions/check.py

@@ -402,13 +402,15 @@ def collect_spot_check_source_paths(
     )
     working_directory = borgmatic.config.paths.get_working_directory(config)
 
-    paths_output = borgmatic.execute.execute_command_and_capture_output(
-        create_flags + create_positional_arguments,
-        capture_stderr=True,
-        environment=borgmatic.borg.environment.make_environment(config),
-        working_directory=working_directory,
-        borg_local_path=local_path,
-        borg_exit_codes=config.get('borg_exit_codes'),
+    paths_output = '\n'.join(
+        borgmatic.execute.execute_command_and_capture_output(
+            create_flags + create_positional_arguments,
+            capture_stderr=True,
+            environment=borgmatic.borg.environment.make_environment(config),
+            working_directory=working_directory,
+            borg_local_path=local_path,
+            borg_exit_codes=config.get('borg_exit_codes'),
+        )
     )
 
     paths = tuple(
@@ -523,15 +525,19 @@ def compare_spot_check_hashes(
         if not source_sample_paths_subset:
             break
 
-        hash_output = borgmatic.execute.execute_command_and_capture_output(
-            tuple(
-                shlex.quote(part)
-                for part in shlex.split(spot_check_config.get('xxh64sum_command', 'xxh64sum'))
+        hash_output = '\n'.join(
+            borgmatic.execute.execute_command_and_capture_output(
+                tuple(
+                    shlex.quote(part)
+                    for part in shlex.split(spot_check_config.get('xxh64sum_command', 'xxh64sum'))
+                )
+                + tuple(
+                    path
+                    for path in source_sample_paths_subset
+                    if path in hashable_source_sample_path
+                ),
+                working_directory=working_directory,
             )
-            + tuple(
-                path for path in source_sample_paths_subset if path in hashable_source_sample_path
-            ),
-            working_directory=working_directory,
         )
 
         source_hashes.update(

borgmatic/borg/create.py

@@ -53,6 +53,7 @@ def validate_planned_backup_paths(
     local_path,
     working_directory,
     borgmatic_runtime_directory,
+    find_special_files=False,
 ):
     '''
     Given a dry-run flag, a Borg create command as a tuple, a configuration dict, a local Borg path,
@@ -61,6 +62,9 @@ def validate_planned_backup_paths(
     given runtime directory exists, validate that it will be included in a backup and hasn't been
     excluded.
 
+    If find special files is True, then return the subset of planned backup paths that are special
+    files. Otherwise, return an empty tuple.
+
     Raise ValueError if the runtime directory has been excluded via "exclude_patterns" or similar,
     because any features that rely on the runtime directory getting backed up will break. For
     instance, without the runtime directory, Borg can't consume any database dumps and borgmatic may
@@ -69,7 +73,7 @@ def validate_planned_backup_paths(
     # Omit "--exclude-nodump" from the Borg dry run command, because that flag causes Borg to open
     # files including any named pipe we've created. And omit "--filter" because that can break the
     # paths output parsing below such that path lines no longer start with the expected "- ".
-    paths_output = execute_command_and_capture_output(
+    path_lines = execute_command_and_capture_output(
         (
             *flags.omit_flag_and_value(
                 flags.omit_flag(
@@ -90,9 +94,9 @@ def validate_planned_backup_paths(
 
     # These are all the individual files that Borg is planning to backup as determined by the Borg
     # create dry run above.
-    paths = tuple(
+    paths = (
         path_line.split(' ', 1)[1]
-        for path_line in paths_output.split('\n')
+        for path_line in path_lines
         if path_line and path_line.startswith(('- ', '+ '))
     )
 
@@ -111,28 +115,34 @@ def validate_planned_backup_paths(
         if pattern.path not in include_pattern_paths
     )
 
-    # If all root patterns in the runtime directory are missing from the paths Borg is planning to
-    # backup, then they must've gotten excluded, e.g. by user-configured excludes. Warn accordingly.
-    if (
-        not dry_run
-        and os.path.exists(borgmatic_runtime_directory)
-        and runtime_directory_root_patterns
-        and not any(
+    special_paths = []
+    runtime_directory_exists = os.path.exists(borgmatic_runtime_directory)
+    candidate_for_warning = bool(
+        not dry_run and runtime_directory_exists and runtime_directory_root_patterns
+    )
+
+    # Do everything in this one loop because we only want to consume the paths generator once.
+    for path in paths:
+        # If all root patterns in the runtime directory are missing from the paths Borg is planning to
+        # backup, then they must've gotten excluded, e.g. by user-configured excludes. Warn accordingly.
+        if candidate_for_warning and not any(
             any_parent_directories(path, (pattern.path,))
             for pattern in runtime_directory_root_patterns
-            for path in paths
-        )
-    ):
-        logger.warning(
-            f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes (or the snapshotted source directories are empty). Please ensure the runtime directory is not excluded.'
-        )
+        ):
+            logger.warning(
+                f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes (or the snapshotted source directories are empty). Please ensure the runtime directory is not excluded.'
+            )
+            candidate_for_warning = False
 
-    # Return the subset of output paths *not* contained within the borgmatic runtime directory. The
-    # intent is that any downstream checks using these paths should skip runtime paths that
-    # borgmatic uses for its own bookkeeping, instead focusing on user-configured paths.
-    return tuple(
-        path for path in paths if not any_parent_directories(path, (borgmatic_runtime_directory,))
-    )
+        # Return the subset of output paths that are special files but *not* contained within the
+        # borgmatic runtime directory. The intent is to skip runtime paths that borgmatic uses for its
+        # own bookkeeping, instead focusing on user-configured paths.
+        if not any_parent_directories(path, (borgmatic_runtime_directory,)) and special_file(
+            path, working_directory
+        ):
+            special_paths.append(path)
+
+    return tuple(special_paths)
 
 
 MAX_SPECIAL_FILE_PATHS_LENGTH = 1000
@@ -253,55 +263,53 @@ def make_base_create_command(  # noqa: PLR0912
         logger.warning(
             'Skipping pre-backup path validation due to "unsafe_skip_path_validation_before_create" option.'
         )
-        planned_backup_paths = ()
-    else:
-        logger.debug('Checking file paths Borg plans to include')
-        planned_backup_paths = validate_planned_backup_paths(
-            dry_run,
-            create_flags + create_positional_arguments,
-            config,
-            patterns,
-            local_path,
-            working_directory,
-            borgmatic_runtime_directory=borgmatic_runtime_directory,
-        )
+
+        return (create_flags, create_positional_arguments, patterns_file)
+
+    logger.debug('Checking file paths Borg plans to include')
+
+    special_file_paths = validate_planned_backup_paths(
+        dry_run,
+        create_flags + create_positional_arguments,
+        config,
+        patterns,
+        local_path,
+        working_directory,
+        borgmatic_runtime_directory=borgmatic_runtime_directory,
+        find_special_files=bool(stream_processes and not config.get('read_special')),
+    )
 
     # If database hooks are enabled (as indicated by streaming processes), exclude files that might
     # cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True.
-    if stream_processes and not config.get('read_special'):
+    if special_file_paths:
         logger.warning(
             'Ignoring configured "read_special" value of false, as true is needed for database hooks.',
         )
 
-        special_file_paths = tuple(
-            path for path in planned_backup_paths if special_file(path, working_directory)
+        truncated_special_file_paths = textwrap.shorten(
+            ', '.join(special_file_paths),
+            width=MAX_SPECIAL_FILE_PATHS_LENGTH,
+            placeholder=' ...',
+        )
+        logger.warning(
+            f'Excluding special files to prevent Borg from hanging: {truncated_special_file_paths}',
+        )
+        patterns_file = borgmatic.borg.pattern.write_patterns_file(
+            tuple(
+                borgmatic.borg.pattern.Pattern(
+                    special_file_path,
+                    borgmatic.borg.pattern.Pattern_type.NO_RECURSE,
+                    borgmatic.borg.pattern.Pattern_style.FNMATCH,
+                    source=borgmatic.borg.pattern.Pattern_source.INTERNAL,
+                )
+                for special_file_path in special_file_paths
+            ),
+            borgmatic_runtime_directory,
+            patterns_file=patterns_file,
         )
 
-        if special_file_paths:
-            truncated_special_file_paths = textwrap.shorten(
-                ', '.join(special_file_paths),
-                width=MAX_SPECIAL_FILE_PATHS_LENGTH,
-                placeholder=' ...',
-            )
-            logger.warning(
-                f'Excluding special files to prevent Borg from hanging: {truncated_special_file_paths}',
-            )
-            patterns_file = borgmatic.borg.pattern.write_patterns_file(
-                tuple(
-                    borgmatic.borg.pattern.Pattern(
-                        special_file_path,
-                        borgmatic.borg.pattern.Pattern_type.NO_RECURSE,
-                        borgmatic.borg.pattern.Pattern_style.FNMATCH,
-                        source=borgmatic.borg.pattern.Pattern_source.INTERNAL,
-                    )
-                    for special_file_path in special_file_paths
-                ),
-                borgmatic_runtime_directory,
-                patterns_file=patterns_file,
-            )
-
-            if '--patterns-from' not in create_flags:
-                create_flags += ('--patterns-from', patterns_file.name)
+        if '--patterns-from' not in create_flags:
+            create_flags += ('--patterns-from', patterns_file.name)
 
     return (create_flags, create_positional_arguments, patterns_file)
 
@@ -385,12 +393,14 @@ def create_archive(
         )
 
     if output_log_level is None:
-        return execute_command_and_capture_output(
-            create_flags + create_positional_arguments,
-            working_directory=working_directory,
-            environment=environment.make_environment(config),
-            borg_local_path=local_path,
-            borg_exit_codes=borg_exit_codes,
+        return '\n'.join(
+            execute_command_and_capture_output(
+                create_flags + create_positional_arguments,
+                working_directory=working_directory,
+                environment=environment.make_environment(config),
+                borg_local_path=local_path,
+                borg_exit_codes=borg_exit_codes,
+            )
         )
 
     execute_command(

borgmatic/borg/info.py

@@ -103,12 +103,14 @@ def display_archives_info(
     borg_exit_codes = config.get('borg_exit_codes')
     working_directory = borgmatic.config.paths.get_working_directory(config)
 
-    json_info = execute_command_and_capture_output(
-        json_command,
-        environment=environment.make_environment(config),
-        working_directory=working_directory,
-        borg_local_path=local_path,
-        borg_exit_codes=borg_exit_codes,
+    json_info = '\n'.join(
+        execute_command_and_capture_output(
+            json_command,
+            environment=environment.make_environment(config),
+            working_directory=working_directory,
+            borg_local_path=local_path,
+            borg_exit_codes=borg_exit_codes,
+        )
     )
 
     if info_arguments.json:

borgmatic/borg/list.py

@@ -116,27 +116,29 @@ def capture_archive_listing(
     output of listing that archive and return it as a list of file paths.
     '''
     return tuple(
-        execute_command_and_capture_output(
-            make_list_command(
-                repository_path,
-                config,
-                local_borg_version,
-                argparse.Namespace(
-                    repository=repository_path,
-                    archive=archive,
-                    paths=list(list_paths) if list_paths else None,
-                    find_paths=None,
-                    json=None,
-                    format=path_format or '{path}{NUL}',
+        ''.join(
+            execute_command_and_capture_output(
+                make_list_command(
+                    repository_path,
+                    config,
+                    local_borg_version,
+                    argparse.Namespace(
+                        repository=repository_path,
+                        archive=archive,
+                        paths=list(list_paths) if list_paths else None,
+                        find_paths=None,
+                        json=None,
+                        format=path_format or '{path}{NUL}',
+                    ),
+                    global_arguments,
+                    local_path,
+                    remote_path,
                 ),
-                global_arguments,
-                local_path,
-                remote_path,
-            ),
-            environment=environment.make_environment(config),
-            working_directory=borgmatic.config.paths.get_working_directory(config),
-            borg_local_path=local_path,
-            borg_exit_codes=config.get('borg_exit_codes'),
+                environment=environment.make_environment(config),
+                working_directory=borgmatic.config.paths.get_working_directory(config),
+                borg_local_path=local_path,
+                borg_exit_codes=config.get('borg_exit_codes'),
+            )
         )
         .strip('\0')
         .split('\0'),
@@ -235,8 +237,6 @@ def list_archive(
                 borg_local_path=local_path,
                 borg_exit_codes=borg_exit_codes,
             )
-            .strip('\n')
-            .splitlines(),
         )
     else:
         archive_lines = (list_arguments.archive,)

borgmatic/borg/passcommand.py

@@ -17,9 +17,11 @@ def run_passcommand(passcommand, working_directory):
     Cache the results so that the passcommand only needs to run—and potentially prompt the user—once
     per borgmatic invocation.
     '''
-    return borgmatic.execute.execute_command_and_capture_output(
-        shlex.split(passcommand),
-        working_directory=working_directory,
+    return '\n'.join(
+        borgmatic.execute.execute_command_and_capture_output(
+            shlex.split(passcommand),
+            working_directory=working_directory,
+        )
     )
 
 

borgmatic/borg/repo_info.py

@@ -60,12 +60,14 @@ def display_repository_info(
     borg_exit_codes = config.get('borg_exit_codes')
 
     if repo_info_arguments.json:
-        return execute_command_and_capture_output(
-            full_command,
-            environment=environment.make_environment(config),
-            working_directory=working_directory,
-            borg_local_path=local_path,
-            borg_exit_codes=borg_exit_codes,
+        return '\n'.join(
+            execute_command_and_capture_output(
+                full_command,
+                environment=environment.make_environment(config),
+                working_directory=working_directory,
+                borg_local_path=local_path,
+                borg_exit_codes=borg_exit_codes,
+            )
         )
 
     execute_command(

borgmatic/borg/repo_list.py

@@ -90,12 +90,14 @@ def get_latest_archive(
         *flags.make_repository_flags(repository_path, local_borg_version),
     )
 
-    json_output = execute_command_and_capture_output(
-        full_command,
-        environment=environment.make_environment(config),
-        working_directory=borgmatic.config.paths.get_working_directory(config),
-        borg_local_path=local_path,
-        borg_exit_codes=config.get('borg_exit_codes'),
+    json_output = '\n'.join(
+        execute_command_and_capture_output(
+            full_command,
+            environment=environment.make_environment(config),
+            working_directory=borgmatic.config.paths.get_working_directory(config),
+            borg_local_path=local_path,
+            borg_exit_codes=config.get('borg_exit_codes'),
+        )
     )
 
     archives = json.loads(json_output)['archives']
@@ -217,12 +219,14 @@ def list_repository(
     working_directory = borgmatic.config.paths.get_working_directory(config)
     borg_exit_codes = config.get('borg_exit_codes')
 
-    json_listing = execute_command_and_capture_output(
-        json_command,
-        environment=environment.make_environment(config),
-        working_directory=working_directory,
-        borg_local_path=local_path,
-        borg_exit_codes=borg_exit_codes,
+    json_listing = '\n'.join(
+        execute_command_and_capture_output(
+            json_command,
+            environment=environment.make_environment(config),
+            working_directory=working_directory,
+            borg_local_path=local_path,
+            borg_exit_codes=borg_exit_codes,
+        )
     )
 
     if repo_list_arguments.json:

borgmatic/borg/version.py

@@ -20,12 +20,14 @@ def local_borg_version(config, local_path='borg'):
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
     )
 
-    output = execute_command_and_capture_output(
-        full_command,
-        environment=environment.make_environment(config),
-        working_directory=borgmatic.config.paths.get_working_directory(config),
-        borg_local_path=local_path,
-        borg_exit_codes=config.get('borg_exit_codes'),
+    output = '\n'.join(
+        execute_command_and_capture_output(
+            full_command,
+            environment=environment.make_environment(config),
+            working_directory=borgmatic.config.paths.get_working_directory(config),
+            borg_local_path=local_path,
+            borg_exit_codes=config.get('borg_exit_codes'),
+        )
     )
 
     try:

borgmatic/execute.py

@@ -153,11 +153,12 @@ def parse_log_line(line, log_level, came_from_stderr, borg_local_path, command):
     return log_line_to_record(line, log_level)
 
 
-def handle_log_record(log_record, last_lines, captured_output):
+def handle_log_record(log_record, last_lines):
     '''
-    Given a log record to be logged, a rolling list of last lines, and a list of captured output,
-    append the record's message to the last lines and (if its log level is None) the captured
-    output. Then log the line.
+    Given a log record to be logged and a rolling list of last lines, append the record's message to
+    the last lines. Then (if the log level is not None), log the record.
+
+    Return the log record.
     '''
     log_message = log_record.getMessage()
     last_lines.append(log_message)
@@ -165,11 +166,10 @@ def handle_log_record(log_record, last_lines, captured_output):
     if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT:
         last_lines.pop(0)
 
-    if log_record.levelno is None:
-        captured_output.append(log_message)
-        return
+    if log_record.levelno is not None:
+        logger.handle(log_record)
 
-    logger.handle(log_record)
+    return log_record
 
 
 def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, borg_exit_codes):  # noqa: PLR0912
@@ -179,9 +179,11 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b
     log level. Additionally, raise a CalledProcessError if a process exits with an error (or a
     warning for exit code 1, if that process does not match the Borg local path).
 
-    If the output log level is None, then instead of logging, capture output for each process and
-    return it as a dict from the process to its output. If the output log level is not None, return
-    an empty dict.
+    If the output log level is None, then instead of logging, capture the output for the last
+    process given and yield it one line at a time. But if the output log level is not None, don't
+    yield anything.
+
+    This yielding means that this function is a generator, and must be consumed in order to execute.
 
     Use the given Borg local path and exit code configuration to decide what's an error and what's a
     warning. If any stdouts are given to exclude, then for any matching processes, ignore those
@@ -197,7 +199,7 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b
         for buffer in output_buffers_for_process(process, exclude_stdouts)
     }
     output_buffers = list(process_for_output_buffer.keys())
-    captured_outputs = collections.defaultdict(list)
+    process_to_capture = processes[-1]
     still_running = True
 
     # Log output for each process until they all exit.
@@ -234,7 +236,7 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b
 
                     # Keep the last few lines of output in case the process errors and we need the
                     # output for the exception below.
-                    handle_log_record(
+                    log_record = handle_log_record(
                         parse_log_line(
                             line=line,
                             log_level=output_log_level,
@@ -243,9 +245,11 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b
                             command=command,
                         ),
                         last_lines=buffer_last_lines[ready_buffer],
-                        captured_output=captured_outputs[ready_process],
                     )
 
+                    if log_record.levelno is None and ready_process == process_to_capture:
+                        yield log_record.getMessage()
+
         if not still_running:
             break
 
@@ -274,7 +278,7 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b
                         if not line:
                             break
 
-                        handle_log_record(
+                        log_record = handle_log_record(
                             parse_log_line(
                                 line=line,
                                 log_level=output_log_level,
@@ -283,9 +287,11 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b
                                 command=command,
                             ),
                             last_lines=last_lines,
-                            captured_output=captured_outputs[process],
                         )
 
+                        if log_record.levelno is None and process == process_to_capture:
+                            yield log_record.getMessage()
+
                 if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT:
                     last_lines.insert(0, '...')
 
@@ -306,13 +312,6 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b
                 still_running = False
                 break
 
-    if output_log_level is None:
-        return {
-            process: '\n'.join(output_lines) for process, output_lines in captured_outputs.items()
-        }
-
-    return {}
-
 
 SECRET_COMMAND_FLAG_NAMES = {'--password'}
 
@@ -415,12 +414,14 @@ def execute_command(
         return process
 
     with borgmatic.logger.Log_prefix(None):  # Log command output without any prefix.
-        log_outputs(
-            (process,),
-            (input_file, output_file),
-            output_log_level,
-            borg_local_path,
-            borg_exit_codes,
+        tuple(
+            log_outputs(
+                (process,),
+                (input_file, output_file),
+                output_log_level,
+                borg_local_path,
+                borg_exit_codes,
+            )
         )
 
     return None
@@ -439,14 +440,16 @@ def execute_command_and_capture_output(
 ):
     '''
     Execute the given command (a sequence of command/argument strings), capturing and returning its
-    output (stdout) as a string. If an input file descriptor is given, then pipe it to the command's
-    stdin. If capture stderr is True, then capture and return stderr in addition to stdout. If shell
-    is True, execute the command within a shell. If an environment variables dict is given, then
-    pass it into the command. If a working directory is given, use that as the present working
-    directory when running the command. If a Borg local path is given, and the command matches it
-    (regardless of arguments), treat exit code 1 as a warning instead of an error. But if Borg exit
-    codes are given as a sequence of exit code configuration dicts, then use that configuration to
-    decide what's an error and what's a warning.
+    output (stdout) as a generator that yields a line at a time. The generator must be consumed in
+    order for the called command to execute.
+
+    If an input file descriptor is given, then pipe it to the command's stdin. If capture stderr is
+    True, then capture stderr in addition to stdout. If shell is True, execute the command within a
+    shell. If an environment variables dict is given, then pass it into the command. If a working
+    directory is given, use that as the present working directory when running the command. If a
+    Borg local path is given, and the command matches it (regardless of arguments), treat exit code
+    1 as a warning instead of an error. But if Borg exit codes are given as a sequence of exit code
+    configuration dicts, then use that configuration to decide what's an error and what's a warning.
 
     Raise subprocesses.CalledProcessError if an error occurs while running the command.
     '''
@@ -474,7 +477,7 @@ def execute_command_and_capture_output(
         return error.output.decode() if error.output is not None else None
 
     with borgmatic.logger.Log_prefix(None):  # Log command output without any prefix.
-        captured_outputs = log_outputs(
+        captured_lines = log_outputs(
             (process,),
             (input_file,),
             None,
@@ -482,7 +485,7 @@ def execute_command_and_capture_output(
             borg_exit_codes,
         )
 
-    return captured_outputs.get(process, '')
+    return captured_lines
 
 
 def execute_command_with_processes(
@@ -543,15 +546,17 @@ def execute_command_with_processes(
         raise
 
     with borgmatic.logger.Log_prefix(None):  # Log command output without any prefix.
-        captured_outputs = log_outputs(
-            (*processes, command_process),
-            (input_file, output_file),
-            output_log_level,
-            borg_local_path,
-            borg_exit_codes,
+        captured_lines = tuple(
+            log_outputs(
+                (*processes, command_process),
+                (input_file, output_file),
+                output_log_level,
+                borg_local_path,
+                borg_exit_codes,
+            )
         )
 
     if output_log_level is None:
-        return captured_outputs.get(command_process, '')
+        return '\n'.join(captured_lines)
 
     return None

borgmatic/hooks/credential/keepassxc.py

@@ -43,4 +43,6 @@ def load_credential(hook_config, config, credential_parameters):
         + (expanded_database_path, attribute_name)  # Ensure database and entry are last.
     )
 
-    return borgmatic.execute.execute_command_and_capture_output(command).rstrip(os.linesep)
+    return '\n'.join(
+        borgmatic.execute.execute_command_and_capture_output(command).rstrip(os.linesep)
+    )

borgmatic/hooks/credential/systemd.py

@@ -48,7 +48,9 @@ def load_credential(hook_config, config, credential_parameters):
             ),
         )
 
-        return borgmatic.execute.execute_command_and_capture_output(command).rstrip(os.linesep)
+        return '\n'.join(
+            borgmatic.execute.execute_command_and_capture_output(command).rstrip(os.linesep)
+        )
 
     try:
         with open(

borgmatic/hooks/data_source/btrfs.py

@@ -53,17 +53,19 @@ def get_subvolume_property(btrfs_command, subvolume_path, property_name):
     As a performance optimization, multiple calls to this function with the same arguments are
     cached.
     '''
-    output = borgmatic.execute.execute_command_and_capture_output(
-        (
-            *btrfs_command.split(' '),
-            'property',
-            'get',
-            '-t',  # Type.
-            'subvol',
-            subvolume_path,
-            property_name,
-        ),
-        close_fds=True,
+    output = '\n'.join(
+        borgmatic.execute.execute_command_and_capture_output(
+            (
+                *btrfs_command.split(' '),
+                'property',
+                'get',
+                '-t',  # Type.
+                'subvol',
+                subvolume_path,
+                property_name,
+            ),
+            close_fds=True,
+        )
     )
 
     try:

borgmatic/hooks/data_source/config.py

@@ -72,13 +72,15 @@ def get_ip_from_container(container):
     last_error = None
     for engine in engines:
         try:
-            output = execute_command_and_capture_output(
-                (
-                    engine,
-                    'container',
-                    'inspect',
-                    '--format={{json .NetworkSettings}}',
-                    container,
+            output = '\n'.join(
+                execute_command_and_capture_output(
+                    (
+                        engine,
+                        'container',
+                        'inspect',
+                        '--format={{json .NetworkSettings}}',
+                        container,
+                    )
                 )
             )
         except subprocess.CalledProcessError as error:

borgmatic/hooks/data_source/lvm.py

@@ -44,17 +44,19 @@ def get_logical_volumes(lsblk_command, patterns=None):
     '''
     try:
         devices_info = json.loads(
-            borgmatic.execute.execute_command_and_capture_output(
-                # Use lsblk instead of lvs here because lvs can't show active mounts.
-                (
-                    *lsblk_command.split(' '),
-                    '--output',
-                    'name,path,mountpoint,type',
-                    '--json',
-                    '--list',
+            ''.join(
+                borgmatic.execute.execute_command_and_capture_output(
+                    # Use lsblk instead of lvs here because lvs can't show active mounts.
+                    (
+                        *lsblk_command.split(' '),
+                        '--output',
+                        'name,path,mountpoint,type',
+                        '--json',
+                        '--list',
+                    ),
+                    close_fds=True,
                 ),
-                close_fds=True,
-            ),
+            )
         )
     except json.JSONDecodeError as error:
         raise ValueError(f'Invalid {lsblk_command} JSON output: {error}')
@@ -321,19 +323,21 @@ def get_snapshots(lvs_command, snapshot_name=None):
     '''
     try:
         snapshot_info = json.loads(
-            borgmatic.execute.execute_command_and_capture_output(
-                # Use lvs instead of lsblk here because lsblk can't filter to just snapshots.
-                (
-                    *lvs_command.split(' '),
-                    '--report-format',
-                    'json',
-                    '--options',
-                    'lv_name,lv_path',
-                    '--select',
-                    'lv_attr =~ ^s',  # Filter to just snapshots.
+            '\n'.join(
+                borgmatic.execute.execute_command_and_capture_output(
+                    # Use lvs instead of lsblk here because lsblk can't filter to just snapshots.
+                    (
+                        *lvs_command.split(' '),
+                        '--report-format',
+                        'json',
+                        '--options',
+                        'lv_name,lv_path',
+                        '--select',
+                        'lv_attr =~ ^s',  # Filter to just snapshots.
+                    ),
+                    close_fds=True,
                 ),
-                close_fds=True,
-            ),
+            )
         )
     except json.JSONDecodeError as error:
         raise ValueError(f'Invalid {lvs_command} JSON output: {error}')

borgmatic/hooks/data_source/mariadb.py

@@ -158,10 +158,12 @@ def database_names_to_dump(database, config, username, password, environment, dr
     if skip_names:
         logger.debug(f'Skipping database names: {", ".join(skip_names)}')
 
-    show_output = execute_command_and_capture_output(
-        show_command,
-        environment=environment,
-        working_directory=borgmatic.config.paths.get_working_directory(config),
+    show_output = '\n'.join(
+        execute_command_and_capture_output(
+            show_command,
+            environment=environment,
+            working_directory=borgmatic.config.paths.get_working_directory(config),
+        )
     )
 
     return tuple(

borgmatic/hooks/data_source/mysql.py

@@ -87,10 +87,12 @@ def database_names_to_dump(database, config, username, password, environment, dr
     if skip_names:
         logger.debug(f'Skipping database names: {", ".join(skip_names)}')
 
-    show_output = execute_command_and_capture_output(
-        show_command,
-        environment=environment,
-        working_directory=borgmatic.config.paths.get_working_directory(config),
+    show_output = '\n'.join(
+        execute_command_and_capture_output(
+            show_command,
+            environment=environment,
+            working_directory=borgmatic.config.paths.get_working_directory(config),
+        )
     )
 
     return tuple(

borgmatic/hooks/data_source/postgresql.py

@@ -103,10 +103,12 @@ def database_names_to_dump(database, config, environment, dry_run):
         + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
     )
     logger.debug('Querying for "all" PostgreSQL databases to dump')
-    list_output = execute_command_and_capture_output(
-        list_command,
-        environment=environment,
-        working_directory=borgmatic.config.paths.get_working_directory(config),
+    list_output = '\n'.join(
+        execute_command_and_capture_output(
+            list_command,
+            environment=environment,
+            working_directory=borgmatic.config.paths.get_working_directory(config),
+        )
     )
 
     return tuple(

borgmatic/hooks/data_source/zfs.py

@@ -45,7 +45,7 @@ def get_datasets_to_backup(zfs_command, patterns):
 
     Return the result as a sequence of Dataset instances, sorted by mount point.
     '''
-    list_output = borgmatic.execute.execute_command_and_capture_output(
+    list_lines = borgmatic.execute.execute_command_and_capture_output(
         (
             *zfs_command.split(' '),
             'list',
@@ -65,7 +65,7 @@ def get_datasets_to_backup(zfs_command, patterns):
         datasets = sorted(
             (
                 Dataset(dataset_name, mount_point, (user_property_value == 'auto'), ())
-                for line in list_output.splitlines()
+                for line in list_lines
                 for (dataset_name, mount_point, can_mount, user_property_value) in (
                     line.rstrip().split('\t'),
                 )
@@ -125,7 +125,7 @@ def get_all_dataset_mount_points(zfs_command):
     '''
     Given a ZFS command to run, return all ZFS datasets as a sequence of sorted mount points.
     '''
-    list_output = borgmatic.execute.execute_command_and_capture_output(
+    list_lines = borgmatic.execute.execute_command_and_capture_output(
         (
             *zfs_command.split(' '),
             'list',
@@ -142,7 +142,7 @@ def get_all_dataset_mount_points(zfs_command):
         sorted(
             {
                 mount_point
-                for line in list_output.splitlines()
+                for line in list_lines
                 for mount_point in (line.rstrip(),)
                 if mount_point != 'none'
             },
@@ -344,7 +344,7 @@ def get_all_snapshots(zfs_command):
     Given a ZFS command to run, return all ZFS snapshots as a sequence of full snapshot names of the
     form "dataset@snapshot".
     '''
-    list_output = borgmatic.execute.execute_command_and_capture_output(
+    list_lines = borgmatic.execute.execute_command_and_capture_output(
         (
             *tuple(zfs_command.split(' ')),
             'list',
@@ -357,7 +357,7 @@ def get_all_snapshots(zfs_command):
         close_fds=True,
     )
 
-    return tuple(line.rstrip() for line in list_output.splitlines())
+    return tuple(line.rstrip() for line in list_lines)
 
 
 def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, patterns, dry_run):  # noqa: PLR0912