diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index 631feaca..ee8e6b90 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -402,13 +402,15 @@ def collect_spot_check_source_paths( ) working_directory = borgmatic.config.paths.get_working_directory(config) - paths_output = borgmatic.execute.execute_command_and_capture_output( - create_flags + create_positional_arguments, - capture_stderr=True, - environment=borgmatic.borg.environment.make_environment(config), - working_directory=working_directory, - borg_local_path=local_path, - borg_exit_codes=config.get('borg_exit_codes'), + paths_output = '\n'.join( + borgmatic.execute.execute_command_and_capture_output( + create_flags + create_positional_arguments, + capture_stderr=True, + environment=borgmatic.borg.environment.make_environment(config), + working_directory=working_directory, + borg_local_path=local_path, + borg_exit_codes=config.get('borg_exit_codes'), + ) ) paths = tuple( @@ -523,15 +525,19 @@ def compare_spot_check_hashes( if not source_sample_paths_subset: break - hash_output = borgmatic.execute.execute_command_and_capture_output( - tuple( - shlex.quote(part) - for part in shlex.split(spot_check_config.get('xxh64sum_command', 'xxh64sum')) + hash_output = '\n'.join( + borgmatic.execute.execute_command_and_capture_output( + tuple( + shlex.quote(part) + for part in shlex.split(spot_check_config.get('xxh64sum_command', 'xxh64sum')) + ) + + tuple( + path + for path in source_sample_paths_subset + if path in hashable_source_sample_path + ), + working_directory=working_directory, ) - + tuple( - path for path in source_sample_paths_subset if path in hashable_source_sample_path - ), - working_directory=working_directory, ) source_hashes.update( diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 459156e4..7759bcb1 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -53,6 +53,7 @@ def validate_planned_backup_paths( local_path, working_directory, borgmatic_runtime_directory, + find_special_files=False, ): ''' Given a dry-run flag, a Borg create command as a tuple, a configuration dict, a local Borg path, @@ -61,6 +62,9 @@ def validate_planned_backup_paths( given runtime directory exists, validate that it will be included in a backup and hasn't been excluded. + If find special files is True, then return the subset of planned backup paths that are special + files. Otherwise, return an empty tuple. + Raise ValueError if the runtime directory has been excluded via "exclude_patterns" or similar, because any features that rely on the runtime directory getting backed up will break. For instance, without the runtime directory, Borg can't consume any database dumps and borgmatic may @@ -69,7 +73,7 @@ def validate_planned_backup_paths( # Omit "--exclude-nodump" from the Borg dry run command, because that flag causes Borg to open # files including any named pipe we've created. And omit "--filter" because that can break the # paths output parsing below such that path lines no longer start with the expected "- ". - paths_output = execute_command_and_capture_output( + path_lines = execute_command_and_capture_output( ( *flags.omit_flag_and_value( flags.omit_flag( @@ -90,9 +94,9 @@ def validate_planned_backup_paths( # These are all the individual files that Borg is planning to backup as determined by the Borg # create dry run above. - paths = tuple( + paths = ( path_line.split(' ', 1)[1] - for path_line in paths_output.split('\n') + for path_line in path_lines if path_line and path_line.startswith(('- ', '+ ')) ) @@ -111,28 +115,34 @@ def validate_planned_backup_paths( if pattern.path not in include_pattern_paths ) - # If all root patterns in the runtime directory are missing from the paths Borg is planning to - # backup, then they must've gotten excluded, e.g. by user-configured excludes. Warn accordingly. - if ( - not dry_run - and os.path.exists(borgmatic_runtime_directory) - and runtime_directory_root_patterns - and not any( + special_paths = [] + runtime_directory_exists = os.path.exists(borgmatic_runtime_directory) + candidate_for_warning = bool( + not dry_run and runtime_directory_exists and runtime_directory_root_patterns + ) + + # Do everything in this one loop because we only want to consume the paths generator once. + for path in paths: + # If all root patterns in the runtime directory are missing from the paths Borg is planning to + # backup, then they must've gotten excluded, e.g. by user-configured excludes. Warn accordingly. + if candidate_for_warning and not any( any_parent_directories(path, (pattern.path,)) for pattern in runtime_directory_root_patterns - for path in paths - ) - ): - logger.warning( - f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes (or the snapshotted source directories are empty). Please ensure the runtime directory is not excluded.' - ) + ): + logger.warning( + f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes (or the snapshotted source directories are empty). Please ensure the runtime directory is not excluded.' + ) + candidate_for_warning = False - # Return the subset of output paths *not* contained within the borgmatic runtime directory. The - # intent is that any downstream checks using these paths should skip runtime paths that - # borgmatic uses for its own bookkeeping, instead focusing on user-configured paths. - return tuple( - path for path in paths if not any_parent_directories(path, (borgmatic_runtime_directory,)) - ) + # Return the subset of output paths that are special files but *not* contained within the + # borgmatic runtime directory. The intent is to skip runtime paths that borgmatic uses for its + # own bookkeeping, instead focusing on user-configured paths. + if not any_parent_directories(path, (borgmatic_runtime_directory,)) and special_file( + path, working_directory + ): + special_paths.append(path) + + return tuple(special_paths) MAX_SPECIAL_FILE_PATHS_LENGTH = 1000 @@ -253,55 +263,53 @@ def make_base_create_command( # noqa: PLR0912 logger.warning( 'Skipping pre-backup path validation due to "unsafe_skip_path_validation_before_create" option.' ) - planned_backup_paths = () - else: - logger.debug('Checking file paths Borg plans to include') - planned_backup_paths = validate_planned_backup_paths( - dry_run, - create_flags + create_positional_arguments, - config, - patterns, - local_path, - working_directory, - borgmatic_runtime_directory=borgmatic_runtime_directory, - ) + + return (create_flags, create_positional_arguments, patterns_file) + + logger.debug('Checking file paths Borg plans to include') + + special_file_paths = validate_planned_backup_paths( + dry_run, + create_flags + create_positional_arguments, + config, + patterns, + local_path, + working_directory, + borgmatic_runtime_directory=borgmatic_runtime_directory, + find_special_files=bool(stream_processes and not config.get('read_special')), + ) # If database hooks are enabled (as indicated by streaming processes), exclude files that might # cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True. - if stream_processes and not config.get('read_special'): + if special_file_paths: logger.warning( 'Ignoring configured "read_special" value of false, as true is needed for database hooks.', ) - special_file_paths = tuple( - path for path in planned_backup_paths if special_file(path, working_directory) + truncated_special_file_paths = textwrap.shorten( + ', '.join(special_file_paths), + width=MAX_SPECIAL_FILE_PATHS_LENGTH, + placeholder=' ...', + ) + logger.warning( + f'Excluding special files to prevent Borg from hanging: {truncated_special_file_paths}', + ) + patterns_file = borgmatic.borg.pattern.write_patterns_file( + tuple( + borgmatic.borg.pattern.Pattern( + special_file_path, + borgmatic.borg.pattern.Pattern_type.NO_RECURSE, + borgmatic.borg.pattern.Pattern_style.FNMATCH, + source=borgmatic.borg.pattern.Pattern_source.INTERNAL, + ) + for special_file_path in special_file_paths + ), + borgmatic_runtime_directory, + patterns_file=patterns_file, ) - if special_file_paths: - truncated_special_file_paths = textwrap.shorten( - ', '.join(special_file_paths), - width=MAX_SPECIAL_FILE_PATHS_LENGTH, - placeholder=' ...', - ) - logger.warning( - f'Excluding special files to prevent Borg from hanging: {truncated_special_file_paths}', - ) - patterns_file = borgmatic.borg.pattern.write_patterns_file( - tuple( - borgmatic.borg.pattern.Pattern( - special_file_path, - borgmatic.borg.pattern.Pattern_type.NO_RECURSE, - borgmatic.borg.pattern.Pattern_style.FNMATCH, - source=borgmatic.borg.pattern.Pattern_source.INTERNAL, - ) - for special_file_path in special_file_paths - ), - borgmatic_runtime_directory, - patterns_file=patterns_file, - ) - - if '--patterns-from' not in create_flags: - create_flags += ('--patterns-from', patterns_file.name) + if '--patterns-from' not in create_flags: + create_flags += ('--patterns-from', patterns_file.name) return (create_flags, create_positional_arguments, patterns_file) @@ -385,12 +393,14 @@ def create_archive( ) if output_log_level is None: - return execute_command_and_capture_output( - create_flags + create_positional_arguments, - working_directory=working_directory, - environment=environment.make_environment(config), - borg_local_path=local_path, - borg_exit_codes=borg_exit_codes, + return '\n'.join( + execute_command_and_capture_output( + create_flags + create_positional_arguments, + working_directory=working_directory, + environment=environment.make_environment(config), + borg_local_path=local_path, + borg_exit_codes=borg_exit_codes, + ) ) execute_command( diff --git a/borgmatic/borg/info.py b/borgmatic/borg/info.py index c2d34cd3..00cc253d 100644 --- a/borgmatic/borg/info.py +++ b/borgmatic/borg/info.py @@ -103,12 +103,14 @@ def display_archives_info( borg_exit_codes = config.get('borg_exit_codes') working_directory = borgmatic.config.paths.get_working_directory(config) - json_info = execute_command_and_capture_output( - json_command, - environment=environment.make_environment(config), - working_directory=working_directory, - borg_local_path=local_path, - borg_exit_codes=borg_exit_codes, + json_info = '\n'.join( + execute_command_and_capture_output( + json_command, + environment=environment.make_environment(config), + working_directory=working_directory, + borg_local_path=local_path, + borg_exit_codes=borg_exit_codes, + ) ) if info_arguments.json: diff --git a/borgmatic/borg/list.py b/borgmatic/borg/list.py index eec10a2d..73e026ba 100644 --- a/borgmatic/borg/list.py +++ b/borgmatic/borg/list.py @@ -116,27 +116,29 @@ def capture_archive_listing( output of listing that archive and return it as a list of file paths. ''' return tuple( - execute_command_and_capture_output( - make_list_command( - repository_path, - config, - local_borg_version, - argparse.Namespace( - repository=repository_path, - archive=archive, - paths=list(list_paths) if list_paths else None, - find_paths=None, - json=None, - format=path_format or '{path}{NUL}', + ''.join( + execute_command_and_capture_output( + make_list_command( + repository_path, + config, + local_borg_version, + argparse.Namespace( + repository=repository_path, + archive=archive, + paths=list(list_paths) if list_paths else None, + find_paths=None, + json=None, + format=path_format or '{path}{NUL}', + ), + global_arguments, + local_path, + remote_path, ), - global_arguments, - local_path, - remote_path, - ), - environment=environment.make_environment(config), - working_directory=borgmatic.config.paths.get_working_directory(config), - borg_local_path=local_path, - borg_exit_codes=config.get('borg_exit_codes'), + environment=environment.make_environment(config), + working_directory=borgmatic.config.paths.get_working_directory(config), + borg_local_path=local_path, + borg_exit_codes=config.get('borg_exit_codes'), + ) ) .strip('\0') .split('\0'), @@ -235,8 +237,6 @@ def list_archive( borg_local_path=local_path, borg_exit_codes=borg_exit_codes, ) - .strip('\n') - .splitlines(), ) else: archive_lines = (list_arguments.archive,) diff --git a/borgmatic/borg/passcommand.py b/borgmatic/borg/passcommand.py index 79132fb4..c93abf1e 100644 --- a/borgmatic/borg/passcommand.py +++ b/borgmatic/borg/passcommand.py @@ -17,9 +17,11 @@ def run_passcommand(passcommand, working_directory): Cache the results so that the passcommand only needs to run—and potentially prompt the user—once per borgmatic invocation. ''' - return borgmatic.execute.execute_command_and_capture_output( - shlex.split(passcommand), - working_directory=working_directory, + return '\n'.join( + borgmatic.execute.execute_command_and_capture_output( + shlex.split(passcommand), + working_directory=working_directory, + ) ) diff --git a/borgmatic/borg/repo_info.py b/borgmatic/borg/repo_info.py index b8c4cfcb..109382bb 100644 --- a/borgmatic/borg/repo_info.py +++ b/borgmatic/borg/repo_info.py @@ -60,12 +60,14 @@ def display_repository_info( borg_exit_codes = config.get('borg_exit_codes') if repo_info_arguments.json: - return execute_command_and_capture_output( - full_command, - environment=environment.make_environment(config), - working_directory=working_directory, - borg_local_path=local_path, - borg_exit_codes=borg_exit_codes, + return '\n'.join( + execute_command_and_capture_output( + full_command, + environment=environment.make_environment(config), + working_directory=working_directory, + borg_local_path=local_path, + borg_exit_codes=borg_exit_codes, + ) ) execute_command( diff --git a/borgmatic/borg/repo_list.py b/borgmatic/borg/repo_list.py index 6af3d5e9..446aacad 100644 --- a/borgmatic/borg/repo_list.py +++ b/borgmatic/borg/repo_list.py @@ -90,12 +90,14 @@ def get_latest_archive( *flags.make_repository_flags(repository_path, local_borg_version), ) - json_output = execute_command_and_capture_output( - full_command, - environment=environment.make_environment(config), - working_directory=borgmatic.config.paths.get_working_directory(config), - borg_local_path=local_path, - borg_exit_codes=config.get('borg_exit_codes'), + json_output = '\n'.join( + execute_command_and_capture_output( + full_command, + environment=environment.make_environment(config), + working_directory=borgmatic.config.paths.get_working_directory(config), + borg_local_path=local_path, + borg_exit_codes=config.get('borg_exit_codes'), + ) ) archives = json.loads(json_output)['archives'] @@ -217,12 +219,14 @@ def list_repository( working_directory = borgmatic.config.paths.get_working_directory(config) borg_exit_codes = config.get('borg_exit_codes') - json_listing = execute_command_and_capture_output( - json_command, - environment=environment.make_environment(config), - working_directory=working_directory, - borg_local_path=local_path, - borg_exit_codes=borg_exit_codes, + json_listing = '\n'.join( + execute_command_and_capture_output( + json_command, + environment=environment.make_environment(config), + working_directory=working_directory, + borg_local_path=local_path, + borg_exit_codes=borg_exit_codes, + ) ) if repo_list_arguments.json: diff --git a/borgmatic/borg/version.py b/borgmatic/borg/version.py index 6bd97690..67f7b002 100644 --- a/borgmatic/borg/version.py +++ b/borgmatic/borg/version.py @@ -20,12 +20,14 @@ def local_borg_version(config, local_path='borg'): + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) ) - output = execute_command_and_capture_output( - full_command, - environment=environment.make_environment(config), - working_directory=borgmatic.config.paths.get_working_directory(config), - borg_local_path=local_path, - borg_exit_codes=config.get('borg_exit_codes'), + output = '\n'.join( + execute_command_and_capture_output( + full_command, + environment=environment.make_environment(config), + working_directory=borgmatic.config.paths.get_working_directory(config), + borg_local_path=local_path, + borg_exit_codes=config.get('borg_exit_codes'), + ) ) try: diff --git a/borgmatic/execute.py b/borgmatic/execute.py index 629ad946..bcc54cad 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -153,11 +153,12 @@ def parse_log_line(line, log_level, came_from_stderr, borg_local_path, command): return log_line_to_record(line, log_level) -def handle_log_record(log_record, last_lines, captured_output): +def handle_log_record(log_record, last_lines): ''' - Given a log record to be logged, a rolling list of last lines, and a list of captured output, - append the record's message to the last lines and (if its log level is None) the captured - output. Then log the line. + Given a log record to be logged and a rolling list of last lines, append the record's message to + the last lines. Then (if the log level is not None), log the record. + + Return the log record. ''' log_message = log_record.getMessage() last_lines.append(log_message) @@ -165,11 +166,10 @@ def handle_log_record(log_record, last_lines, captured_output): if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT: last_lines.pop(0) - if log_record.levelno is None: - captured_output.append(log_message) - return + if log_record.levelno is not None: + logger.handle(log_record) - logger.handle(log_record) + return log_record def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, borg_exit_codes): # noqa: PLR0912 @@ -179,9 +179,11 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b log level. Additionally, raise a CalledProcessError if a process exits with an error (or a warning for exit code 1, if that process does not match the Borg local path). - If the output log level is None, then instead of logging, capture output for each process and - return it as a dict from the process to its output. If the output log level is not None, return - an empty dict. + If the output log level is None, then instead of logging, capture the output for the last + process given and yield it one line at a time. But if the output log level is not None, don't + yield anything. + + This yielding means that this function is a generator, and must be consumed in order to execute. Use the given Borg local path and exit code configuration to decide what's an error and what's a warning. If any stdouts are given to exclude, then for any matching processes, ignore those @@ -197,7 +199,7 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b for buffer in output_buffers_for_process(process, exclude_stdouts) } output_buffers = list(process_for_output_buffer.keys()) - captured_outputs = collections.defaultdict(list) + process_to_capture = processes[-1] still_running = True # Log output for each process until they all exit. @@ -234,7 +236,7 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b # Keep the last few lines of output in case the process errors and we need the # output for the exception below. - handle_log_record( + log_record = handle_log_record( parse_log_line( line=line, log_level=output_log_level, @@ -243,9 +245,11 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b command=command, ), last_lines=buffer_last_lines[ready_buffer], - captured_output=captured_outputs[ready_process], ) + if log_record.levelno is None and ready_process == process_to_capture: + yield log_record.getMessage() + if not still_running: break @@ -274,7 +278,7 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b if not line: break - handle_log_record( + log_record = handle_log_record( parse_log_line( line=line, log_level=output_log_level, @@ -283,9 +287,11 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b command=command, ), last_lines=last_lines, - captured_output=captured_outputs[process], ) + if log_record.levelno is None and process == process_to_capture: + yield log_record.getMessage() + if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT: last_lines.insert(0, '...') @@ -306,13 +312,6 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b still_running = False break - if output_log_level is None: - return { - process: '\n'.join(output_lines) for process, output_lines in captured_outputs.items() - } - - return {} - SECRET_COMMAND_FLAG_NAMES = {'--password'} @@ -415,12 +414,14 @@ def execute_command( return process with borgmatic.logger.Log_prefix(None): # Log command output without any prefix. - log_outputs( - (process,), - (input_file, output_file), - output_log_level, - borg_local_path, - borg_exit_codes, + tuple( + log_outputs( + (process,), + (input_file, output_file), + output_log_level, + borg_local_path, + borg_exit_codes, + ) ) return None @@ -439,14 +440,16 @@ def execute_command_and_capture_output( ): ''' Execute the given command (a sequence of command/argument strings), capturing and returning its - output (stdout) as a string. If an input file descriptor is given, then pipe it to the command's - stdin. If capture stderr is True, then capture and return stderr in addition to stdout. If shell - is True, execute the command within a shell. If an environment variables dict is given, then - pass it into the command. If a working directory is given, use that as the present working - directory when running the command. If a Borg local path is given, and the command matches it - (regardless of arguments), treat exit code 1 as a warning instead of an error. But if Borg exit - codes are given as a sequence of exit code configuration dicts, then use that configuration to - decide what's an error and what's a warning. + output (stdout) as a generator that yields a line at a time. The generator must be consumed in + order for the called command to execute. + + If an input file descriptor is given, then pipe it to the command's stdin. If capture stderr is + True, then capture stderr in addition to stdout. If shell is True, execute the command within a + shell. If an environment variables dict is given, then pass it into the command. If a working + directory is given, use that as the present working directory when running the command. If a + Borg local path is given, and the command matches it (regardless of arguments), treat exit code + 1 as a warning instead of an error. But if Borg exit codes are given as a sequence of exit code + configuration dicts, then use that configuration to decide what's an error and what's a warning. Raise subprocesses.CalledProcessError if an error occurs while running the command. ''' @@ -474,7 +477,7 @@ def execute_command_and_capture_output( return error.output.decode() if error.output is not None else None with borgmatic.logger.Log_prefix(None): # Log command output without any prefix. - captured_outputs = log_outputs( + captured_lines = log_outputs( (process,), (input_file,), None, @@ -482,7 +485,7 @@ def execute_command_and_capture_output( borg_exit_codes, ) - return captured_outputs.get(process, '') + return captured_lines def execute_command_with_processes( @@ -543,15 +546,17 @@ def execute_command_with_processes( raise with borgmatic.logger.Log_prefix(None): # Log command output without any prefix. - captured_outputs = log_outputs( - (*processes, command_process), - (input_file, output_file), - output_log_level, - borg_local_path, - borg_exit_codes, + captured_lines = tuple( + log_outputs( + (*processes, command_process), + (input_file, output_file), + output_log_level, + borg_local_path, + borg_exit_codes, + ) ) if output_log_level is None: - return captured_outputs.get(command_process, '') + return '\n'.join(captured_lines) return None diff --git a/borgmatic/hooks/credential/keepassxc.py b/borgmatic/hooks/credential/keepassxc.py index 9a8f46a1..0638e5fd 100644 --- a/borgmatic/hooks/credential/keepassxc.py +++ b/borgmatic/hooks/credential/keepassxc.py @@ -43,4 +43,6 @@ def load_credential(hook_config, config, credential_parameters): + (expanded_database_path, attribute_name) # Ensure database and entry are last. ) - return borgmatic.execute.execute_command_and_capture_output(command).rstrip(os.linesep) + return '\n'.join( + borgmatic.execute.execute_command_and_capture_output(command).rstrip(os.linesep) + ) diff --git a/borgmatic/hooks/credential/systemd.py b/borgmatic/hooks/credential/systemd.py index 7213f396..40d98fb5 100644 --- a/borgmatic/hooks/credential/systemd.py +++ b/borgmatic/hooks/credential/systemd.py @@ -48,7 +48,9 @@ def load_credential(hook_config, config, credential_parameters): ), ) - return borgmatic.execute.execute_command_and_capture_output(command).rstrip(os.linesep) + return '\n'.join( + borgmatic.execute.execute_command_and_capture_output(command).rstrip(os.linesep) + ) try: with open( diff --git a/borgmatic/hooks/data_source/btrfs.py b/borgmatic/hooks/data_source/btrfs.py index 9f938c75..2f2fb6ee 100644 --- a/borgmatic/hooks/data_source/btrfs.py +++ b/borgmatic/hooks/data_source/btrfs.py @@ -53,17 +53,19 @@ def get_subvolume_property(btrfs_command, subvolume_path, property_name): As a performance optimization, multiple calls to this function with the same arguments are cached. ''' - output = borgmatic.execute.execute_command_and_capture_output( - ( - *btrfs_command.split(' '), - 'property', - 'get', - '-t', # Type. - 'subvol', - subvolume_path, - property_name, - ), - close_fds=True, + output = '\n'.join( + borgmatic.execute.execute_command_and_capture_output( + ( + *btrfs_command.split(' '), + 'property', + 'get', + '-t', # Type. + 'subvol', + subvolume_path, + property_name, + ), + close_fds=True, + ) ) try: diff --git a/borgmatic/hooks/data_source/config.py b/borgmatic/hooks/data_source/config.py index 98ab5590..f4954674 100644 --- a/borgmatic/hooks/data_source/config.py +++ b/borgmatic/hooks/data_source/config.py @@ -72,13 +72,15 @@ def get_ip_from_container(container): last_error = None for engine in engines: try: - output = execute_command_and_capture_output( - ( - engine, - 'container', - 'inspect', - '--format={{json .NetworkSettings}}', - container, + output = '\n'.join( + execute_command_and_capture_output( + ( + engine, + 'container', + 'inspect', + '--format={{json .NetworkSettings}}', + container, + ) ) ) except subprocess.CalledProcessError as error: diff --git a/borgmatic/hooks/data_source/lvm.py b/borgmatic/hooks/data_source/lvm.py index a51834af..545430b3 100644 --- a/borgmatic/hooks/data_source/lvm.py +++ b/borgmatic/hooks/data_source/lvm.py @@ -44,17 +44,19 @@ def get_logical_volumes(lsblk_command, patterns=None): ''' try: devices_info = json.loads( - borgmatic.execute.execute_command_and_capture_output( - # Use lsblk instead of lvs here because lvs can't show active mounts. - ( - *lsblk_command.split(' '), - '--output', - 'name,path,mountpoint,type', - '--json', - '--list', + ''.join( + borgmatic.execute.execute_command_and_capture_output( + # Use lsblk instead of lvs here because lvs can't show active mounts. + ( + *lsblk_command.split(' '), + '--output', + 'name,path,mountpoint,type', + '--json', + '--list', + ), + close_fds=True, ), - close_fds=True, - ), + ) ) except json.JSONDecodeError as error: raise ValueError(f'Invalid {lsblk_command} JSON output: {error}') @@ -321,19 +323,21 @@ def get_snapshots(lvs_command, snapshot_name=None): ''' try: snapshot_info = json.loads( - borgmatic.execute.execute_command_and_capture_output( - # Use lvs instead of lsblk here because lsblk can't filter to just snapshots. - ( - *lvs_command.split(' '), - '--report-format', - 'json', - '--options', - 'lv_name,lv_path', - '--select', - 'lv_attr =~ ^s', # Filter to just snapshots. + '\n'.join( + borgmatic.execute.execute_command_and_capture_output( + # Use lvs instead of lsblk here because lsblk can't filter to just snapshots. + ( + *lvs_command.split(' '), + '--report-format', + 'json', + '--options', + 'lv_name,lv_path', + '--select', + 'lv_attr =~ ^s', # Filter to just snapshots. + ), + close_fds=True, ), - close_fds=True, - ), + ) ) except json.JSONDecodeError as error: raise ValueError(f'Invalid {lvs_command} JSON output: {error}') diff --git a/borgmatic/hooks/data_source/mariadb.py b/borgmatic/hooks/data_source/mariadb.py index 190a0d7c..18884681 100644 --- a/borgmatic/hooks/data_source/mariadb.py +++ b/borgmatic/hooks/data_source/mariadb.py @@ -158,10 +158,12 @@ def database_names_to_dump(database, config, username, password, environment, dr if skip_names: logger.debug(f'Skipping database names: {", ".join(skip_names)}') - show_output = execute_command_and_capture_output( - show_command, - environment=environment, - working_directory=borgmatic.config.paths.get_working_directory(config), + show_output = '\n'.join( + execute_command_and_capture_output( + show_command, + environment=environment, + working_directory=borgmatic.config.paths.get_working_directory(config), + ) ) return tuple( diff --git a/borgmatic/hooks/data_source/mysql.py b/borgmatic/hooks/data_source/mysql.py index ab412312..7cbe4108 100644 --- a/borgmatic/hooks/data_source/mysql.py +++ b/borgmatic/hooks/data_source/mysql.py @@ -87,10 +87,12 @@ def database_names_to_dump(database, config, username, password, environment, dr if skip_names: logger.debug(f'Skipping database names: {", ".join(skip_names)}') - show_output = execute_command_and_capture_output( - show_command, - environment=environment, - working_directory=borgmatic.config.paths.get_working_directory(config), + show_output = '\n'.join( + execute_command_and_capture_output( + show_command, + environment=environment, + working_directory=borgmatic.config.paths.get_working_directory(config), + ) ) return tuple( diff --git a/borgmatic/hooks/data_source/postgresql.py b/borgmatic/hooks/data_source/postgresql.py index af2cc987..93463980 100644 --- a/borgmatic/hooks/data_source/postgresql.py +++ b/borgmatic/hooks/data_source/postgresql.py @@ -103,10 +103,12 @@ def database_names_to_dump(database, config, environment, dry_run): + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ()) ) logger.debug('Querying for "all" PostgreSQL databases to dump') - list_output = execute_command_and_capture_output( - list_command, - environment=environment, - working_directory=borgmatic.config.paths.get_working_directory(config), + list_output = '\n'.join( + execute_command_and_capture_output( + list_command, + environment=environment, + working_directory=borgmatic.config.paths.get_working_directory(config), + ) ) return tuple( diff --git a/borgmatic/hooks/data_source/zfs.py b/borgmatic/hooks/data_source/zfs.py index 7b285849..40ae25d4 100644 --- a/borgmatic/hooks/data_source/zfs.py +++ b/borgmatic/hooks/data_source/zfs.py @@ -45,7 +45,7 @@ def get_datasets_to_backup(zfs_command, patterns): Return the result as a sequence of Dataset instances, sorted by mount point. ''' - list_output = borgmatic.execute.execute_command_and_capture_output( + list_lines = borgmatic.execute.execute_command_and_capture_output( ( *zfs_command.split(' '), 'list', @@ -65,7 +65,7 @@ def get_datasets_to_backup(zfs_command, patterns): datasets = sorted( ( Dataset(dataset_name, mount_point, (user_property_value == 'auto'), ()) - for line in list_output.splitlines() + for line in list_lines for (dataset_name, mount_point, can_mount, user_property_value) in ( line.rstrip().split('\t'), ) @@ -125,7 +125,7 @@ def get_all_dataset_mount_points(zfs_command): ''' Given a ZFS command to run, return all ZFS datasets as a sequence of sorted mount points. ''' - list_output = borgmatic.execute.execute_command_and_capture_output( + list_lines = borgmatic.execute.execute_command_and_capture_output( ( *zfs_command.split(' '), 'list', @@ -142,7 +142,7 @@ def get_all_dataset_mount_points(zfs_command): sorted( { mount_point - for line in list_output.splitlines() + for line in list_lines for mount_point in (line.rstrip(),) if mount_point != 'none' }, @@ -344,7 +344,7 @@ def get_all_snapshots(zfs_command): Given a ZFS command to run, return all ZFS snapshots as a sequence of full snapshot names of the form "dataset@snapshot". ''' - list_output = borgmatic.execute.execute_command_and_capture_output( + list_lines = borgmatic.execute.execute_command_and_capture_output( ( *tuple(zfs_command.split(' ')), 'list', @@ -357,7 +357,7 @@ def get_all_snapshots(zfs_command): close_fds=True, ) - return tuple(line.rstrip() for line in list_output.splitlines()) + return tuple(line.rstrip() for line in list_lines) def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, patterns, dry_run): # noqa: PLR0912