peter/borgmatic
1
0
Fork 0
mirror of https://github.com/borgmatic-collective/borgmatic.git synced 2026-07-22 02:03:01 +02:00
Code Issues Packages Projects Releases Wiki Activity

Reduce memory consumption of "create" action path validation (#1225).

Browse Source 
Reviewed-on: https://projects.torsion.org/borgmatic-collective/borgmatic/pulls/1227
This commit is contained in:
Dan Helfman
2026-01-09 21:29:07 +00:00
parent 3e4115cc92 6c7af8c2bb
commit a5f7c8cafc
42 changed files with 1078 additions and 823 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEWS
+4 -2
View File
@@ -17,9 +17,11 @@
resolve it as a constant/variable.
* #1220: Cleanup snapshots immediately after ZFS, LVM, or Btrfs hooks error—rather than waiting
until the next time borgmatic runs.
* #1221: Add an "unsafe_skip_path_validation_before_create" option to skip pre-backup safety checks
so as to reduce backup times on large filesystems.
* #1221: Add an "unsafe_skip_path_validation_before_create" option to skip pre-backup safety
validation so as to reduce backup times on large filesystems.
* #1224: When running an "extract" check with the "--progress" flag, show file extraction progress.
* #1225: Improve performance and greatly reduce memory usage during pre-backup safety validation on
large filesystems.
* When syslog verbosity is enabled, log to systemd's journal (if present) with
structured data. See the documentation for more information:
https://torsion.org/borgmatic/reference/command-line/logging/#systemd-journal
borgmatic/actions/check.py
+5 -5
View File
@@ -402,7 +402,7 @@ def collect_spot_check_source_paths(
)
working_directory = borgmatic.config.paths.get_working_directory(config)
paths_output = borgmatic.execute.execute_command_and_capture_output(
path_lines = borgmatic.execute.execute_command_and_capture_output(
create_flags + create_positional_arguments,
capture_stderr=True,
environment=borgmatic.borg.environment.make_environment(config),
@@ -411,9 +411,9 @@ def collect_spot_check_source_paths(
borg_exit_codes=config.get('borg_exit_codes'),
)
paths = tuple(
paths = (
path_line.split(' ', 1)[1]
for path_line in paths_output.splitlines()
for path_line in path_lines
if path_line and path_line.startswith(('- ', '+ '))
)
@@ -523,7 +523,7 @@ def compare_spot_check_hashes(
if not source_sample_paths_subset:
break
hash_output = borgmatic.execute.execute_command_and_capture_output(
hash_lines = borgmatic.execute.execute_command_and_capture_output(
tuple(
shlex.quote(part)
for part in shlex.split(spot_check_config.get('xxh64sum_command', 'xxh64sum'))
@@ -536,7 +536,7 @@ def compare_spot_check_hashes(
source_hashes.update(
**dict(
(reversed(line.split(' ', 1)) for line in hash_output.splitlines()),
(reversed(line.split(' ', 1)) for line in hash_lines),
# Represent non-existent files as having empty hashes so the comparison below still
# works. Same thing for filesystem links, since Borg produces empty archive hashes
# for them.
borgmatic/borg/create.py
+93 -73
View File
@@ -53,6 +53,7 @@ def validate_planned_backup_paths(
local_path,
working_directory,
borgmatic_runtime_directory,
find_special_files=False,
):
'''
Given a dry-run flag, a Borg create command as a tuple, a configuration dict, a local Borg path,
@@ -61,6 +62,9 @@ def validate_planned_backup_paths(
given runtime directory exists, validate that it will be included in a backup and hasn't been
excluded.
If find special files is True, then return the subset of planned backup paths that are special
files. Otherwise, return an empty tuple.
Raise ValueError if the runtime directory has been excluded via "exclude_patterns" or similar,
because any features that rely on the runtime directory getting backed up will break. For
instance, without the runtime directory, Borg can't consume any database dumps and borgmatic may
@@ -69,7 +73,7 @@ def validate_planned_backup_paths(
# Omit "--exclude-nodump" from the Borg dry run command, because that flag causes Borg to open
# files including any named pipe we've created. And omit "--filter" because that can break the
# paths output parsing below such that path lines no longer start with the expected "- ".
paths_output = execute_command_and_capture_output(
path_lines = execute_command_and_capture_output(
(
*flags.omit_flag_and_value(
flags.omit_flag(
@@ -90,9 +94,9 @@ def validate_planned_backup_paths(
# These are all the individual files that Borg is planning to backup as determined by the Borg
# create dry run above.
paths = tuple(
paths = (
path_line.split(' ', 1)[1]
for path_line in paths_output.split('\n')
for path_line in path_lines
if path_line and path_line.startswith(('- ', '+ '))
)
@@ -111,28 +115,41 @@ def validate_planned_backup_paths(
if pattern.path not in include_pattern_paths
)
# If all root patterns in the runtime directory are missing from the paths Borg is planning to
# backup, then they must've gotten excluded, e.g. by user-configured excludes. Warn accordingly.
if (
special_paths = []
validate_runtime_directory = bool(
not dry_run
and os.path.exists(borgmatic_runtime_directory)
and runtime_directory_root_patterns
and not any(
)
runtime_directory_in_path = False
# Do everything in this one loop because we only want to consume the paths generator once.
for path in paths:
# If all root patterns in the runtime directory are missing from the paths Borg is planning
# to backup, then they must've gotten excluded, e.g. by user-configured excludes. Warn
# accordingly (below).
if validate_runtime_directory and any(
any_parent_directories(path, (pattern.path,))
for pattern in runtime_directory_root_patterns
for path in paths
)
):
):
runtime_directory_in_path = True
# Return the subset of output paths that are special files but *not* contained within the
# borgmatic runtime directory. The intent is to skip runtime paths that borgmatic uses for its
# own bookkeeping, instead focusing on user-configured paths.
if (
find_special_files
and not any_parent_directories(path, (borgmatic_runtime_directory,))
and special_file(path, working_directory)
):
special_paths.append(path)
if validate_runtime_directory and not runtime_directory_in_path:
logger.warning(
f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes (or the snapshotted source directories are empty). Please ensure the runtime directory is not excluded.'
)
# Return the subset of output paths *not* contained within the borgmatic runtime directory. The
# intent is that any downstream checks using these paths should skip runtime paths that
# borgmatic uses for its own bookkeeping, instead focusing on user-configured paths.
return tuple(
path for path in paths if not any_parent_directories(path, (borgmatic_runtime_directory,))
)
return tuple(special_paths)
MAX_SPECIAL_FILE_PATHS_LENGTH = 1000
@@ -253,55 +270,54 @@ def make_base_create_command( # noqa: PLR0912
logger.warning(
'Skipping pre-backup path validation due to "unsafe_skip_path_validation_before_create" option.'
)
planned_backup_paths = ()
else:
logger.debug('Checking file paths Borg plans to include')
planned_backup_paths = validate_planned_backup_paths(
dry_run,
create_flags + create_positional_arguments,
config,
patterns,
local_path,
working_directory,
borgmatic_runtime_directory=borgmatic_runtime_directory,
)
# If database hooks are enabled (as indicated by streaming processes), exclude files that might
# cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True.
if stream_processes and not config.get('read_special'):
return (create_flags, create_positional_arguments, patterns_file)
logger.debug('Checking file paths Borg plans to include')
special_file_paths = validate_planned_backup_paths(
dry_run,
create_flags + create_positional_arguments,
config,
patterns,
local_path,
working_directory,
borgmatic_runtime_directory=borgmatic_runtime_directory,
find_special_files=bool(stream_processes),
)
if stream_processes and config.get('read_special') is False:
logger.warning(
'Ignoring configured "read_special" value of false, as true is needed for database hooks.',
)
special_file_paths = tuple(
path for path in planned_backup_paths if special_file(path, working_directory)
# If database hooks are enabled (as indicated by streaming processes), exclude files that might
# cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True.
if special_file_paths:
truncated_special_file_paths = textwrap.shorten(
', '.join(special_file_paths),
width=MAX_SPECIAL_FILE_PATHS_LENGTH,
placeholder=' ...',
)
logger.warning(
f'Excluding special files to prevent Borg from hanging: {truncated_special_file_paths}',
)
patterns_file = borgmatic.borg.pattern.write_patterns_file(
tuple(
borgmatic.borg.pattern.Pattern(
special_file_path,
borgmatic.borg.pattern.Pattern_type.NO_RECURSE,
borgmatic.borg.pattern.Pattern_style.FNMATCH,
source=borgmatic.borg.pattern.Pattern_source.INTERNAL,
)
for special_file_path in special_file_paths
),
borgmatic_runtime_directory,
patterns_file=patterns_file,
)
if special_file_paths:
truncated_special_file_paths = textwrap.shorten(
', '.join(special_file_paths),
width=MAX_SPECIAL_FILE_PATHS_LENGTH,
placeholder=' ...',
)
logger.warning(
f'Excluding special files to prevent Borg from hanging: {truncated_special_file_paths}',
)
patterns_file = borgmatic.borg.pattern.write_patterns_file(
tuple(
borgmatic.borg.pattern.Pattern(
special_file_path,
borgmatic.borg.pattern.Pattern_type.NO_RECURSE,
borgmatic.borg.pattern.Pattern_style.FNMATCH,
source=borgmatic.borg.pattern.Pattern_source.INTERNAL,
)
for special_file_path in special_file_paths
),
borgmatic_runtime_directory,
patterns_file=patterns_file,
)
if '--patterns-from' not in create_flags:
create_flags += ('--patterns-from', patterns_file.name)
if '--patterns-from' not in create_flags:
create_flags += ('--patterns-from', patterns_file.name)
return (create_flags, create_positional_arguments, patterns_file)
@@ -373,24 +389,28 @@ def create_archive(
borg_exit_codes = config.get('borg_exit_codes')
if stream_processes:
return execute_command_with_processes(
create_flags + create_positional_arguments,
stream_processes,
output_log_level,
output_file,
working_directory=working_directory,
environment=environment.make_environment(config),
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
return '\n'.join(
execute_command_with_processes(
create_flags + create_positional_arguments,
stream_processes,
output_log_level,
output_file,
working_directory=working_directory,
environment=environment.make_environment(config),
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
)
)
if output_log_level is None:
return execute_command_and_capture_output(
create_flags + create_positional_arguments,
working_directory=working_directory,
environment=environment.make_environment(config),
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
return '\n'.join(
execute_command_and_capture_output(
create_flags + create_positional_arguments,
working_directory=working_directory,
environment=environment.make_environment(config),
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
)
)
execute_command(
borgmatic/borg/info.py
+8 -6
View File
@@ -103,12 +103,14 @@ def display_archives_info(
borg_exit_codes = config.get('borg_exit_codes')
working_directory = borgmatic.config.paths.get_working_directory(config)
json_info = execute_command_and_capture_output(
json_command,
environment=environment.make_environment(config),
working_directory=working_directory,
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
json_info = '\n'.join(
execute_command_and_capture_output(
json_command,
environment=environment.make_environment(config),
working_directory=working_directory,
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
)
)
if info_arguments.json:
borgmatic/borg/list.py
+36 -38
View File
@@ -116,27 +116,29 @@ def capture_archive_listing(
output of listing that archive and return it as a list of file paths.
'''
return tuple(
execute_command_and_capture_output(
make_list_command(
repository_path,
config,
local_borg_version,
argparse.Namespace(
repository=repository_path,
archive=archive,
paths=list(list_paths) if list_paths else None,
find_paths=None,
json=None,
format=path_format or '{path}{NUL}',
''.join(
execute_command_and_capture_output(
make_list_command(
repository_path,
config,
local_borg_version,
argparse.Namespace(
repository=repository_path,
archive=archive,
paths=list(list_paths) if list_paths else None,
find_paths=None,
json=None,
format=path_format or '{path}{NUL}',
),
global_arguments,
local_path,
remote_path,
),
global_arguments,
local_path,
remote_path,
),
environment=environment.make_environment(config),
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
environment=environment.make_environment(config),
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)
)
.strip('\0')
.split('\0'),
@@ -219,24 +221,20 @@ def list_archive(
)
# Ask Borg to list archives. Capture its output for use below.
archive_lines = tuple(
execute_command_and_capture_output(
repo_list.make_repo_list_command(
repository_path,
config,
local_borg_version,
repo_list_arguments,
global_arguments,
local_path,
remote_path,
),
environment=environment.make_environment(config),
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
)
.strip('\n')
.splitlines(),
archive_lines = execute_command_and_capture_output(
repo_list.make_repo_list_command(
repository_path,
config,
local_borg_version,
repo_list_arguments,
global_arguments,
local_path,
remote_path,
),
environment=environment.make_environment(config),
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
)
else:
archive_lines = (list_arguments.archive,)
borgmatic/borg/passcommand.py
+5 -3
View File
@@ -17,9 +17,11 @@ def run_passcommand(passcommand, working_directory):
Cache the results so that the passcommand only needs to run—and potentially prompt the user—once
per borgmatic invocation.
'''
return borgmatic.execute.execute_command_and_capture_output(
shlex.split(passcommand),
working_directory=working_directory,
return '\n'.join(
borgmatic.execute.execute_command_and_capture_output(
shlex.split(passcommand),
working_directory=working_directory,
)
)
borgmatic/borg/repo_info.py
+8 -6
View File
@@ -59,12 +59,14 @@ def display_repository_info(
borg_exit_codes = config.get('borg_exit_codes')
if repo_info_arguments.json:
return execute_command_and_capture_output(
full_command,
environment=environment.make_environment(config),
working_directory=working_directory,
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
return '\n'.join(
execute_command_and_capture_output(
full_command,
environment=environment.make_environment(config),
working_directory=working_directory,
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
)
)
execute_command(
borgmatic/borg/repo_list.py
+16 -12
View File
@@ -90,12 +90,14 @@ def get_latest_archive(
*flags.make_repository_flags(repository_path, local_borg_version),
)
json_output = execute_command_and_capture_output(
full_command,
environment=environment.make_environment(config),
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
json_output = '\n'.join(
execute_command_and_capture_output(
full_command,
environment=environment.make_environment(config),
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)
)
archives = json.loads(json_output)['archives']
@@ -217,12 +219,14 @@ def list_repository(
working_directory = borgmatic.config.paths.get_working_directory(config)
borg_exit_codes = config.get('borg_exit_codes')
json_listing = execute_command_and_capture_output(
json_command,
environment=environment.make_environment(config),
working_directory=working_directory,
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
json_listing = '\n'.join(
execute_command_and_capture_output(
json_command,
environment=environment.make_environment(config),
working_directory=working_directory,
borg_local_path=local_path,
borg_exit_codes=borg_exit_codes,
)
)
if repo_list_arguments.json:
borgmatic/borg/version.py
+8 -6
View File
@@ -20,12 +20,14 @@ def local_borg_version(config, local_path='borg'):
+ (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
)
output = execute_command_and_capture_output(
full_command,
environment=environment.make_environment(config),
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
output = '\n'.join(
execute_command_and_capture_output(
full_command,
environment=environment.make_environment(config),
working_directory=borgmatic.config.paths.get_working_directory(config),
borg_local_path=local_path,
borg_exit_codes=config.get('borg_exit_codes'),
)
)
try:
borgmatic/execute.py
+58 -52
View File
@@ -153,11 +153,12 @@ def parse_log_line(line, log_level, came_from_stderr, borg_local_path, command):
return log_line_to_record(line, log_level)
def handle_log_record(log_record, last_lines, captured_output):
def handle_log_record(log_record, last_lines):
'''
Given a log record to be logged, a rolling list of last lines, and a list of captured output,
append the record's message to the last lines and (if its log level is None) the captured
output. Then log the line.
Given a log record to be logged and a rolling list of last lines, append the record's message to
the last lines. Then (if the log level is not None), log the record.
Return the log record.
'''
log_message = log_record.getMessage()
last_lines.append(log_message)
@@ -165,11 +166,10 @@ def handle_log_record(log_record, last_lines, captured_output):
if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT:
last_lines.pop(0)
if log_record.levelno is None:
captured_output.append(log_message)
return
if log_record.levelno is not None:
logger.handle(log_record)
logger.handle(log_record)
return log_record
def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, borg_exit_codes): # noqa: PLR0912
@@ -179,9 +179,11 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b
log level. Additionally, raise a CalledProcessError if a process exits with an error (or a
warning for exit code 1, if that process does not match the Borg local path).
If the output log level is None, then instead of logging, capture output for each process and
return it as a dict from the process to its output. If the output log level is not None, return
an empty dict.
If the output log level is None, then instead of logging, capture the output for the last
process given and yield it one line at a time. But if the output log level is not None, don't
yield anything.
This yielding means that this function is a generator, and must be consumed in order to execute.
Use the given Borg local path and exit code configuration to decide what's an error and what's a
warning. If any stdouts are given to exclude, then for any matching processes, ignore those
@@ -197,7 +199,7 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b
for buffer in output_buffers_for_process(process, exclude_stdouts)
}
output_buffers = list(process_for_output_buffer.keys())
captured_outputs = collections.defaultdict(list)
process_to_capture = processes[-1]
still_running = True
# Log output for each process until they all exit.
@@ -234,7 +236,7 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b
# Keep the last few lines of output in case the process errors and we need the
# output for the exception below.
handle_log_record(
log_record = handle_log_record(
parse_log_line(
line=line,
log_level=output_log_level,
@@ -243,9 +245,11 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b
command=command,
),
last_lines=process_last_lines[ready_process],
captured_output=captured_outputs[ready_process],
)
if log_record.levelno is None and ready_process == process_to_capture:
yield log_record.getMessage()
if not still_running:
break
@@ -274,7 +278,7 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b
if not line:
break
handle_log_record(
log_record = handle_log_record(
parse_log_line(
line=line,
log_level=output_log_level,
@@ -283,9 +287,11 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b
command=command,
),
last_lines=last_lines,
captured_output=captured_outputs[process],
)
if log_record.levelno is None and process == process_to_capture:
yield log_record.getMessage()
if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT:
last_lines.insert(0, '...')
@@ -306,13 +312,6 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path, b
still_running = False
break
if output_log_level is None:
return {
process: '\n'.join(output_lines) for process, output_lines in captured_outputs.items()
}
return {}
SECRET_COMMAND_FLAG_NAMES = {'--password'}
@@ -415,12 +414,14 @@ def execute_command(
return process
with borgmatic.logger.Log_prefix(None): # Log command output without any prefix.
log_outputs(
(process,),
(input_file, output_file),
output_log_level,
borg_local_path,
borg_exit_codes,
tuple(
log_outputs(
(process,),
(input_file, output_file),
output_log_level,
borg_local_path,
borg_exit_codes,
)
)
return None
@@ -439,14 +440,16 @@ def execute_command_and_capture_output(
):
'''
Execute the given command (a sequence of command/argument strings), capturing and returning its
output (stdout) as a string. If an input file descriptor is given, then pipe it to the command's
stdin. If capture stderr is True, then capture and return stderr in addition to stdout. If shell
is True, execute the command within a shell. If an environment variables dict is given, then
pass it into the command. If a working directory is given, use that as the present working
directory when running the command. If a Borg local path is given, and the command matches it
(regardless of arguments), treat exit code 1 as a warning instead of an error. But if Borg exit
codes are given as a sequence of exit code configuration dicts, then use that configuration to
decide what's an error and what's a warning.
output (stdout) as a generator that yields one line at a time. The generator must be consumed in
order for the called command to execute.
If an input file descriptor is given, then pipe it to the command's stdin. If capture stderr is
True, then capture stderr in addition to stdout. If shell is True, execute the command within a
shell. If an environment variables dict is given, then pass it into the command. If a working
directory is given, use that as the present working directory when running the command. If a
Borg local path is given, and the command matches it (regardless of arguments), treat exit code
1 as a warning instead of an error. But if Borg exit codes are given as a sequence of exit code
configuration dicts, then use that configuration to decide what's an error and what's a warning.
Raise subprocesses.CalledProcessError if an error occurs while running the command.
'''
@@ -471,10 +474,13 @@ def execute_command_and_capture_output(
):
raise
return error.output.decode() if error.output is not None else None
if error.output is not None:
yield from iter(error.output.decode().splitlines())
return
with borgmatic.logger.Log_prefix(None): # Log command output without any prefix.
captured_outputs = log_outputs(
captured_lines = log_outputs(
(process,),
(input_file,),
None,
@@ -482,7 +488,7 @@ def execute_command_and_capture_output(
borg_exit_codes,
)
return captured_outputs.get(process, '')
yield from captured_lines
def execute_command_with_processes(
@@ -506,13 +512,16 @@ def execute_command_with_processes(
If an open output file object is given, then write stdout to the file and only log stderr. But
if output log level is None, instead suppress logging and return the captured output for (only)
the given command. If an open input file object is given, then read stdin from the file. If
shell is True, execute the command within a shell. If an environment variables dict is given,
then pass it into the command. If a working directory is given, use that as the present working
directory when running the command. If a Borg local path is given, then for any matching command
or process (regardless of arguments), treat exit code 1 as a warning instead of an error. But if
Borg exit codes are given as a sequence of exit code configuration dicts, then use that
configuration to decide what's an error and what's a warning.
the given command as a generator that yields one line at a time. The generator must be consumed
in order for the called command to execute—regardless of the output log level.
If an open input file object is given, then read stdin from the file. If shell is True, execute
the command within a shell. If an environment variables dict is given, then pass it into the
command. If a working directory is given, use that as the present working directory when running
the command. If a Borg local path is given, then for any matching command or process (regardless
of arguments), treat exit code 1 as a warning instead of an error. But if Borg exit codes are
given as a sequence of exit code configuration dicts, then use that configuration to decide
what's an error and what's a warning.
Raise subprocesses.CalledProcessError if an error occurs while running the command or in the
upstream process.
@@ -543,7 +552,7 @@ def execute_command_with_processes(
raise
with borgmatic.logger.Log_prefix(None): # Log command output without any prefix.
captured_outputs = log_outputs(
captured_lines = log_outputs(
(*processes, command_process),
(input_file, output_file),
output_log_level,
@@ -551,7 +560,4 @@ def execute_command_with_processes(
borg_exit_codes,
)
if output_log_level is None:
return captured_outputs.get(command_process, '')
return None
yield from captured_lines
borgmatic/hooks/credential/keepassxc.py
+3 -1
View File
@@ -43,4 +43,6 @@ def load_credential(hook_config, config, credential_parameters):
+ (expanded_database_path, attribute_name) # Ensure database and entry are last.
)
return borgmatic.execute.execute_command_and_capture_output(command).rstrip(os.linesep)
return '\n'.join(borgmatic.execute.execute_command_and_capture_output(command)).rstrip(
os.linesep
)
borgmatic/hooks/credential/systemd.py
+3 -1
View File
@@ -48,7 +48,9 @@ def load_credential(hook_config, config, credential_parameters):
),
)
return borgmatic.execute.execute_command_and_capture_output(command).rstrip(os.linesep)
return '\n'.join(borgmatic.execute.execute_command_and_capture_output(command)).rstrip(
os.linesep
)
try:
with open(
borgmatic/hooks/data_source/btrfs.py
+13 -11
View File
@@ -53,17 +53,19 @@ def get_subvolume_property(btrfs_command, subvolume_path, property_name):
As a performance optimization, multiple calls to this function with the same arguments are
cached.
'''
output = borgmatic.execute.execute_command_and_capture_output(
(
*btrfs_command.split(' '),
'property',
'get',
'-t', # Type.
'subvol',
subvolume_path,
property_name,
),
close_fds=True,
output = '\n'.join(
borgmatic.execute.execute_command_and_capture_output(
(
*btrfs_command.split(' '),
'property',
'get',
'-t', # Type.
'subvol',
subvolume_path,
property_name,
),
close_fds=True,
)
)
try:
borgmatic/hooks/data_source/config.py
+9 -7
View File
@@ -72,13 +72,15 @@ def get_ip_from_container(container):
last_error = None
for engine in engines:
try:
output = execute_command_and_capture_output(
(
engine,
'container',
'inspect',
'--format={{json .NetworkSettings}}',
container,
output = '\n'.join(
execute_command_and_capture_output(
(
engine,
'container',
'inspect',
'--format={{json .NetworkSettings}}',
container,
)
)
)
except subprocess.CalledProcessError as error:
borgmatic/hooks/data_source/lvm.py
+26 -22
View File
@@ -44,17 +44,19 @@ def get_logical_volumes(lsblk_command, patterns=None):
'''
try:
devices_info = json.loads(
borgmatic.execute.execute_command_and_capture_output(
# Use lsblk instead of lvs here because lvs can't show active mounts.
(
*lsblk_command.split(' '),
'--output',
'name,path,mountpoint,type',
'--json',
'--list',
'\n'.join(
borgmatic.execute.execute_command_and_capture_output(
# Use lsblk instead of lvs here because lvs can't show active mounts.
(
*lsblk_command.split(' '),
'--output',
'name,path,mountpoint,type',
'--json',
'--list',
),
close_fds=True,
),
close_fds=True,
),
)
)
except json.JSONDecodeError as error:
raise ValueError(f'Invalid {lsblk_command} JSON output: {error}')
@@ -321,19 +323,21 @@ def get_snapshots(lvs_command, snapshot_name=None):
'''
try:
snapshot_info = json.loads(
borgmatic.execute.execute_command_and_capture_output(
# Use lvs instead of lsblk here because lsblk can't filter to just snapshots.
(
*lvs_command.split(' '),
'--report-format',
'json',
'--options',
'lv_name,lv_path',
'--select',
'lv_attr =~ ^s', # Filter to just snapshots.
'\n'.join(
borgmatic.execute.execute_command_and_capture_output(
# Use lvs instead of lsblk here because lsblk can't filter to just snapshots.
(
*lvs_command.split(' '),
'--report-format',
'json',
'--options',
'lv_name,lv_path',
'--select',
'lv_attr =~ ^s', # Filter to just snapshots.
),
close_fds=True,
),
close_fds=True,
),
)
)
except json.JSONDecodeError as error:
raise ValueError(f'Invalid {lvs_command} JSON output: {error}')
borgmatic/hooks/data_source/mariadb.py
+12 -10
View File
@@ -158,15 +158,15 @@ def database_names_to_dump(database, config, username, password, environment, dr
if skip_names:
logger.debug(f'Skipping database names: {", ".join(skip_names)}')
show_output = execute_command_and_capture_output(
show_lines = execute_command_and_capture_output(
show_command,
environment=environment,
working_directory=borgmatic.config.paths.get_working_directory(config),
)
return tuple(
show_name
for show_name in show_output.strip().splitlines()
show_name.strip()
for show_name in show_lines
if show_name not in SYSTEM_DATABASE_NAMES
if not skip_names or show_name not in skip_names
)
@@ -516,11 +516,13 @@ def restore_data_source_dump(
# Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
# if the restore paths don't exist in the archive.
execute_command_with_processes(
restore_command,
[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
environment=environment,
working_directory=borgmatic.config.paths.get_working_directory(config),
tuple(
execute_command_with_processes(
restore_command,
[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
environment=environment,
working_directory=borgmatic.config.paths.get_working_directory(config),
)
)
borgmatic/hooks/data_source/mongodb.py
+8 -6
View File
@@ -289,12 +289,14 @@ def restore_data_source_dump(
# Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
# if the restore paths don't exist in the archive.
execute_command_with_processes(
restore_command,
[extract_process] if extract_process else [],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout if extract_process else None,
working_directory=borgmatic.config.paths.get_working_directory(config),
tuple(
execute_command_with_processes(
restore_command,
[extract_process] if extract_process else [],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout if extract_process else None,
working_directory=borgmatic.config.paths.get_working_directory(config),
)
)
borgmatic/hooks/data_source/mysql.py
+12 -10
View File
@@ -87,15 +87,15 @@ def database_names_to_dump(database, config, username, password, environment, dr
if skip_names:
logger.debug(f'Skipping database names: {", ".join(skip_names)}')
show_output = execute_command_and_capture_output(
show_lines = execute_command_and_capture_output(
show_command,
environment=environment,
working_directory=borgmatic.config.paths.get_working_directory(config),
)
return tuple(
show_name
for show_name in show_output.strip().splitlines()
show_name.strip()
for show_name in show_lines
if show_name not in SYSTEM_DATABASE_NAMES
if not skip_names or show_name not in skip_names
)
@@ -453,11 +453,13 @@ def restore_data_source_dump(
# Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
# if the restore paths don't exist in the archive.
execute_command_with_processes(
restore_command,
[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
environment=environment,
working_directory=borgmatic.config.paths.get_working_directory(config),
tuple(
execute_command_with_processes(
restore_command,
[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
environment=environment,
working_directory=borgmatic.config.paths.get_working_directory(config),
)
)
borgmatic/hooks/data_source/postgresql.py
+11 -9
View File
@@ -103,7 +103,7 @@ def database_names_to_dump(database, config, environment, dry_run):
+ (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
)
logger.debug('Querying for "all" PostgreSQL databases to dump')
list_output = execute_command_and_capture_output(
list_lines = execute_command_and_capture_output(
list_command,
environment=environment,
working_directory=borgmatic.config.paths.get_working_directory(config),
@@ -111,7 +111,7 @@ def database_names_to_dump(database, config, environment, dry_run):
return tuple(
row[0]
for row in csv.reader(list_output.splitlines(), delimiter=',', quotechar='"')
for row in csv.reader(list_lines, delimiter=',', quotechar='"')
if row[0] not in EXCLUDED_DATABASE_NAMES
)
@@ -434,13 +434,15 @@ def restore_data_source_dump(
# Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
# if the restore paths don't exist in the archive.
execute_command_with_processes(
restore_command,
[extract_process] if extract_process else [],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout if extract_process else None,
environment=environment,
working_directory=borgmatic.config.paths.get_working_directory(config),
tuple(
execute_command_with_processes(
restore_command,
[extract_process] if extract_process else [],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout if extract_process else None,
environment=environment,
working_directory=borgmatic.config.paths.get_working_directory(config),
)
)
execute_command(
analyze_command,
borgmatic/hooks/data_source/sqlite.py
+8 -6
View File
@@ -217,10 +217,12 @@ def restore_data_source_dump(
restore_command = (*sqlite_restore_command, '-bail', shlex.quote(database_path))
# Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
# if the restore paths don't exist in the archive.
execute_command_with_processes(
restore_command,
[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=borgmatic.config.paths.get_working_directory(config),
tuple(
execute_command_with_processes(
restore_command,
[extract_process],
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=borgmatic.config.paths.get_working_directory(config),
)
)
borgmatic/hooks/data_source/zfs.py
+6 -6
View File
@@ -45,7 +45,7 @@ def get_datasets_to_backup(zfs_command, patterns):
Return the result as a sequence of Dataset instances, sorted by mount point.
'''
list_output = borgmatic.execute.execute_command_and_capture_output(
list_lines = borgmatic.execute.execute_command_and_capture_output(
(
*zfs_command.split(' '),
'list',
@@ -65,7 +65,7 @@ def get_datasets_to_backup(zfs_command, patterns):
datasets = sorted(
(
Dataset(dataset_name, mount_point, (user_property_value == 'auto'), ())
for line in list_output.splitlines()
for line in list_lines
for (dataset_name, mount_point, can_mount, user_property_value) in (
line.rstrip().split('\t'),
)
@@ -125,7 +125,7 @@ def get_all_dataset_mount_points(zfs_command):
'''
Given a ZFS command to run, return all ZFS datasets as a sequence of sorted mount points.
'''
list_output = borgmatic.execute.execute_command_and_capture_output(
list_lines = borgmatic.execute.execute_command_and_capture_output(
(
*zfs_command.split(' '),
'list',
@@ -142,7 +142,7 @@ def get_all_dataset_mount_points(zfs_command):
sorted(
{
mount_point
for line in list_output.splitlines()
for line in list_lines
for mount_point in (line.rstrip(),)
if mount_point != 'none'
},
@@ -344,7 +344,7 @@ def get_all_snapshots(zfs_command):
Given a ZFS command to run, return all ZFS snapshots as a sequence of full snapshot names of the
form "dataset@snapshot".
'''
list_output = borgmatic.execute.execute_command_and_capture_output(
list_lines = borgmatic.execute.execute_command_and_capture_output(
(
*tuple(zfs_command.split(' ')),
'list',
@@ -357,7 +357,7 @@ def get_all_snapshots(zfs_command):
close_fds=True,
)
return tuple(line.rstrip() for line in list_output.splitlines())
return tuple(line.rstrip() for line in list_lines)
def remove_data_source_dumps(hook_config, config, borgmatic_runtime_directory, patterns, dry_run): # noqa: PLR0912
tests/integration/test_execute.py
+121 -93
View File
@@ -44,14 +44,16 @@ def test_log_outputs_logs_each_line_separately():
).and_return((there_process.stdout,))
assert (
module.log_outputs(
(hi_process, there_process),
exclude_stdouts=(),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
tuple(
module.log_outputs(
(hi_process, there_process),
exclude_stdouts=(),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
)
)
== {}
== ()
)
@@ -77,14 +79,16 @@ def test_log_outputs_logs_stderr_as_error():
).and_return((echo_process.stdout, echo_process.stderr))
assert (
module.log_outputs(
(echo_process,),
exclude_stdouts=(),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
tuple(
module.log_outputs(
(echo_process,),
exclude_stdouts=(),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
)
)
== {}
== ()
)
@@ -115,14 +119,16 @@ def test_log_outputs_skips_logs_for_process_with_none_stdout():
).and_return((there_process.stdout,))
assert (
module.log_outputs(
(hi_process, there_process),
exclude_stdouts=(),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
tuple(
module.log_outputs(
(hi_process, there_process),
exclude_stdouts=(),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
)
)
== {}
== ()
)
@@ -142,15 +148,17 @@ def test_log_outputs_returns_output_without_logging_for_output_log_level_none():
(),
).and_return((there_process.stdout,))
captured_outputs = module.log_outputs(
(hi_process, there_process),
exclude_stdouts=(),
output_log_level=None,
borg_local_path='borg',
borg_exit_codes=None,
output_lines = tuple(
module.log_outputs(
(hi_process, there_process),
exclude_stdouts=(),
output_log_level=None,
borg_local_path='borg',
borg_exit_codes=None,
)
)
assert captured_outputs == {hi_process: 'hi', there_process: 'there'}
assert output_lines == ('there',)
def test_log_outputs_includes_error_output_in_exception():
@@ -162,12 +170,14 @@ def test_log_outputs_includes_error_output_in_exception():
flexmock(module).should_receive('output_buffers_for_process').and_return((process.stdout,))
with pytest.raises(subprocess.CalledProcessError) as error:
module.log_outputs(
(process,),
exclude_stdouts=(),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
tuple(
module.log_outputs(
(process,),
exclude_stdouts=(),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
)
)
assert error.value.output
@@ -190,12 +200,14 @@ def test_log_outputs_logs_multiline_error_output():
flexmock(module.logger).should_call('handle').at_least().times(3)
with pytest.raises(subprocess.CalledProcessError):
module.log_outputs(
(process,),
exclude_stdouts=(),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
tuple(
module.log_outputs(
(process,),
exclude_stdouts=(),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
)
)
@@ -208,12 +220,14 @@ def test_log_outputs_skips_error_output_in_exception_for_process_with_none_stdou
flexmock(module).should_receive('output_buffers_for_process').and_return((process.stdout,))
with pytest.raises(subprocess.CalledProcessError) as error:
module.log_outputs(
(process,),
exclude_stdouts=(),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
tuple(
module.log_outputs(
(process,),
exclude_stdouts=(),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
)
)
assert error.value.returncode == 2
@@ -258,12 +272,14 @@ def test_log_outputs_kills_other_processes_and_raises_when_one_errors():
flexmock(other_process).should_receive('kill').once()
with pytest.raises(subprocess.CalledProcessError) as error:
module.log_outputs(
(process, other_process),
exclude_stdouts=(),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
tuple(
module.log_outputs(
(process, other_process),
exclude_stdouts=(),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
)
)
assert error.value.returncode == 2
@@ -308,14 +324,16 @@ def test_log_outputs_kills_other_processes_and_returns_when_one_exits_with_warni
flexmock(other_process).should_receive('kill').once()
assert (
module.log_outputs(
(process, other_process),
exclude_stdouts=(),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
tuple(
module.log_outputs(
(process, other_process),
exclude_stdouts=(),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
)
)
== {}
== ()
)
@@ -355,14 +373,16 @@ def test_log_outputs_vents_other_processes_when_one_exits():
flexmock(process.stdout).should_call('readline').at_least().once()
assert (
module.log_outputs(
(process, other_process),
exclude_stdouts=(process.stdout,),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
tuple(
module.log_outputs(
(process, other_process),
exclude_stdouts=(process.stdout,),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
)
)
== {}
== ()
)
@@ -395,14 +415,16 @@ def test_log_outputs_does_not_error_when_one_process_exits():
).and_return((other_process.stdout,))
assert (
module.log_outputs(
(process, other_process),
exclude_stdouts=(process.stdout,),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
tuple(
module.log_outputs(
(process, other_process),
exclude_stdouts=(process.stdout,),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
)
)
== {}
== ()
)
@@ -426,12 +448,14 @@ def test_log_outputs_truncates_long_error_output():
flexmock(module).should_receive('output_buffers_for_process').and_return((process.stdout,))
with pytest.raises(subprocess.CalledProcessError) as error:
flexmock(module, ERROR_OUTPUT_MAX_LINE_COUNT=0).log_outputs(
(process,),
exclude_stdouts=(),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
tuple(
flexmock(module, ERROR_OUTPUT_MAX_LINE_COUNT=0).log_outputs(
(process,),
exclude_stdouts=(),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
)
)
assert error.value.returncode == 2
@@ -446,14 +470,16 @@ def test_log_outputs_with_no_output_logs_nothing():
flexmock(module).should_receive('output_buffers_for_process').and_return((process.stdout,))
assert (
module.log_outputs(
(process,),
exclude_stdouts=(),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
tuple(
module.log_outputs(
(process,),
exclude_stdouts=(),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
)
)
== {}
== ()
)
@@ -466,12 +492,14 @@ def test_log_outputs_with_unfinished_process_re_polls():
flexmock(module).should_receive('output_buffers_for_process').and_return((process.stdout,))
assert (
module.log_outputs(
(process,),
exclude_stdouts=(),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
tuple(
module.log_outputs(
(process,),
exclude_stdouts=(),
output_log_level=logging.INFO,
borg_local_path='borg',
borg_exit_codes=None,
)
)
== {}
== ()
)
tests/unit/actions/test_check.py
+54 -29
View File
@@ -618,8 +618,11 @@ def test_collect_spot_check_source_paths_parses_borg_output_and_includes_bootstr
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return(
'warning: stuff\n- /etc/path\n+ /etc/other\n? /nope',
).and_yield(
'warning: stuff',
'- /etc/path',
'+ /etc/other',
'? /nope',
)
flexmock(module.os.path).should_receive('isfile').and_return(True)
@@ -666,8 +669,11 @@ def test_collect_spot_check_source_paths_omits_progress_from_create_dry_run_comm
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return(
'warning: stuff\n- /etc/path\n+ /etc/other\n? /nope',
).and_yield(
'warning: stuff',
'- /etc/path',
'+ /etc/other',
'? /nope',
)
flexmock(module.os.path).should_receive('isfile').and_return(True)
@@ -714,8 +720,11 @@ def test_collect_spot_check_source_paths_passes_through_stream_processes_false()
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return(
'warning: stuff\n- /etc/path\n+ /etc/other\n? /nope',
).and_yield(
'warning: stuff',
'- /etc/path',
'+ /etc/other',
'? /nope',
)
flexmock(module.os.path).should_receive('isfile').and_return(True)
@@ -762,8 +771,11 @@ def test_collect_spot_check_source_paths_without_working_directory_parses_borg_o
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return(
'warning: stuff\n- /etc/path\n+ /etc/other\n? /nope',
).and_yield(
'warning: stuff',
'- /etc/path',
'+ /etc/other',
'? /nope',
)
flexmock(module.os.path).should_receive('isfile').and_return(True)
@@ -810,8 +822,11 @@ def test_collect_spot_check_source_paths_skips_directories():
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return(
'warning: stuff\n- /etc/path\n+ /etc/dir\n? /nope',
).and_yield(
'warning: stuff',
'- /etc/path',
'+ /etc/dir',
'? /nope',
)
flexmock(module.os.path).should_receive('isfile').with_args('/etc/path').and_return(False)
flexmock(module.os.path).should_receive('isfile').with_args('/etc/dir').and_return(False)
@@ -958,8 +973,11 @@ def test_collect_spot_check_source_paths_uses_working_directory():
)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return(
'warning: stuff\n- foo\n+ bar\n? /nope',
).and_yield(
'warning: stuff',
'- foo',
'+ bar',
'? /nope',
)
flexmock(module.os.path).should_receive('isfile').with_args('/working/dir/foo').and_return(True)
flexmock(module.os.path).should_receive('isfile').with_args('/working/dir/bar').and_return(True)
@@ -987,8 +1005,9 @@ def test_compare_spot_check_hashes_returns_paths_having_failing_hashes():
flexmock(module.os.path).should_receive('islink').and_return(False)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return(
'hash1 /foo\nhash2 /bar',
).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_yield(
'hash1 /foo',
'hash2 /bar',
)
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
['hash1 foo', 'nothash2 bar'],
@@ -1028,8 +1047,9 @@ def test_compare_spot_check_hashes_returns_relative_paths_having_failing_hashes(
flexmock(module.os.path).should_receive('islink').and_return(False)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).with_args(('xxh64sum', 'foo', 'bar'), working_directory=None).and_return(
'hash1 foo\nhash2 bar',
).with_args(('xxh64sum', 'foo', 'bar'), working_directory=None).and_yield(
'hash1 foo',
'hash2 bar',
)
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
['hash1 foo', 'nothash2 bar'],
@@ -1069,8 +1089,9 @@ def test_compare_spot_check_hashes_handles_data_sample_percentage_above_100():
flexmock(module.os.path).should_receive('islink').and_return(False)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return(
'hash1 /foo\nhash2 /bar',
).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_yield(
'hash1 /foo',
'hash2 /bar',
)
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
['nothash1 foo', 'nothash2 bar'],
@@ -1113,7 +1134,7 @@ def test_compare_spot_check_hashes_uses_xxh64sum_command_option():
).with_args(
('/usr/local/bin/xxhsum', '-H64', '/foo', '/bar'),
working_directory=None,
).and_return('hash1 /foo\nhash2 /bar')
).and_yield('hash1 /foo', 'hash2 /bar')
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
['hash1 foo', 'nothash2 bar'],
)
@@ -1149,8 +1170,9 @@ def test_compare_spot_check_hashes_considers_path_missing_from_archive_as_not_ma
flexmock(module.os.path).should_receive('islink').and_return(False)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return(
'hash1 /foo\nhash2 /bar',
).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_yield(
'hash1 /foo',
'hash2 /bar',
)
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
['hash1 foo'],
@@ -1187,7 +1209,7 @@ def test_compare_spot_check_hashes_considers_symlink_path_as_not_matching():
flexmock(module.os.path).should_receive('islink').with_args('/bar').and_return(True)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).with_args(('xxh64sum', '/foo'), working_directory=None).and_return('hash1 /foo')
).with_args(('xxh64sum', '/foo'), working_directory=None).and_yield('hash1 /foo')
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
['hash1 foo', 'hash2 bar'],
)
@@ -1223,7 +1245,7 @@ def test_compare_spot_check_hashes_considers_non_existent_path_as_not_matching()
flexmock(module.os.path).should_receive('islink').and_return(False)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).with_args(('xxh64sum', '/foo'), working_directory=None).and_return('hash1 /foo')
).with_args(('xxh64sum', '/foo'), working_directory=None).and_yield('hash1 /foo')
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
['hash1 foo', 'hash2 bar'],
)
@@ -1259,13 +1281,15 @@ def test_compare_spot_check_hashes_with_too_many_paths_feeds_them_to_commands_in
flexmock(module.os.path).should_receive('islink').and_return(False)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_return(
'hash1 /foo\nhash2 /bar',
).with_args(('xxh64sum', '/foo', '/bar'), working_directory=None).and_yield(
'hash1 /foo',
'hash2 /bar',
)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).with_args(('xxh64sum', '/baz', '/quux'), working_directory=None).and_return(
'hash3 /baz\nhash4 /quux',
).with_args(('xxh64sum', '/baz', '/quux'), working_directory=None).and_yield(
'hash3 /baz',
'hash4 /quux',
)
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
['hash1 foo', 'hash2 bar'],
@@ -1306,8 +1330,9 @@ def test_compare_spot_check_hashes_uses_working_directory_to_access_source_paths
flexmock(module.os.path).should_receive('islink').and_return(False)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).with_args(('xxh64sum', 'foo', 'bar'), working_directory='/working/dir').and_return(
'hash1 foo\nhash2 bar',
).with_args(('xxh64sum', 'foo', 'bar'), working_directory='/working/dir').and_yield(
'hash1 foo',
'hash2 bar',
)
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_return(
['hash1 foo', 'nothash2 bar'],
tests/unit/borg/test_create.py
+268 -147
View File
@@ -67,25 +67,36 @@ def test_validate_planned_backup_paths_parses_borg_dry_run_file_list():
lambda arguments, flag: arguments,
)
flexmock(module.environment).should_receive('make_environment').and_return(None)
flexmock(module).should_receive('execute_command_and_capture_output').and_return(
'Processing files ...\n- /foo\n+ /bar\n- /baz',
flexmock(module).should_receive('execute_command_and_capture_output').and_yield(
'Processing files ...',
'- /foo',
'+ /bar',
'- /baz',
)
flexmock(module.os.path).should_receive('exists').and_return(False)
flexmock(module).should_receive('any_parent_directories').and_return(False)
flexmock(module).should_receive('any_parent_directories').replace_with(
lambda path, candidates: any(path.startswith(parent) for parent in candidates)
)
flexmock(module).should_receive('special_file').with_args('/foo', None).and_return(False)
flexmock(module).should_receive('special_file').with_args('/bar', None).and_return(False)
flexmock(module).should_receive('special_file').with_args('/baz', None).and_return(False)
assert module.validate_planned_backup_paths(
dry_run=False,
create_command=('borg', 'create'),
config={},
patterns=(
module.borgmatic.borg.pattern.Pattern('/foo'),
module.borgmatic.borg.pattern.Pattern('/bar'),
module.borgmatic.borg.pattern.Pattern('/baz'),
),
local_path=None,
working_directory=None,
borgmatic_runtime_directory='/run/borgmatic',
) == ('/foo', '/bar', '/baz')
assert (
module.validate_planned_backup_paths(
dry_run=False,
create_command=('borg', 'create'),
config={},
patterns=(
module.borgmatic.borg.pattern.Pattern('/foo'),
module.borgmatic.borg.pattern.Pattern('/bar'),
module.borgmatic.borg.pattern.Pattern('/baz'),
),
local_path=None,
working_directory=None,
borgmatic_runtime_directory='/run/borgmatic',
)
== ()
)
def test_validate_planned_backup_paths_skips_borgmatic_runtime_directory():
@@ -96,29 +107,37 @@ def test_validate_planned_backup_paths_skips_borgmatic_runtime_directory():
lambda arguments, flag: arguments,
)
flexmock(module.environment).should_receive('make_environment').and_return(None)
flexmock(module).should_receive('execute_command_and_capture_output').and_return(
'+ /foo\n- /run/borgmatic/bar\n- /baz',
flexmock(module).should_receive('execute_command_and_capture_output').and_yield(
'+ /foo',
'- /run/borgmatic/bar',
'- /baz',
)
flexmock(module.os.path).should_receive('exists').and_return(True)
flexmock(module).should_receive('any_parent_directories').replace_with(
lambda path, candidates: any(path.startswith(parent) for parent in candidates)
)
flexmock(module).should_receive('special_file').with_args('/foo', None).and_return(False)
flexmock(module).should_receive('special_file').with_args('/run/borgmatic/bar', None).never()
flexmock(module).should_receive('special_file').with_args('/baz', None).and_return(False)
assert module.validate_planned_backup_paths(
dry_run=False,
create_command=('borg', 'create'),
config={},
patterns=(
module.borgmatic.borg.pattern.Pattern('/foo'),
module.borgmatic.borg.pattern.Pattern(
'/run/borgmatic/bar', module.borgmatic.borg.pattern.Pattern_type.ROOT
assert (
module.validate_planned_backup_paths(
dry_run=False,
create_command=('borg', 'create'),
config={},
patterns=(
module.borgmatic.borg.pattern.Pattern('/foo'),
module.borgmatic.borg.pattern.Pattern(
'/run/borgmatic/bar', module.borgmatic.borg.pattern.Pattern_type.ROOT
),
module.borgmatic.borg.pattern.Pattern('/baz'),
),
module.borgmatic.borg.pattern.Pattern('/baz'),
),
local_path=None,
working_directory=None,
borgmatic_runtime_directory='/run/borgmatic',
) == ('/foo', '/baz')
local_path=None,
working_directory=None,
borgmatic_runtime_directory='/run/borgmatic',
)
== ()
)
def test_validate_planned_backup_paths_with_borgmatic_runtime_directory_missing_from_paths_output_warns():
@@ -129,30 +148,38 @@ def test_validate_planned_backup_paths_with_borgmatic_runtime_directory_missing_
lambda arguments, flag: arguments,
)
flexmock(module.environment).should_receive('make_environment').and_return(None)
flexmock(module).should_receive('execute_command_and_capture_output').and_return(
'+ /foo\n- /bar\n- /baz',
flexmock(module).should_receive('execute_command_and_capture_output').and_yield(
'+ /foo',
'- /bar',
'- /baz',
)
flexmock(module.os.path).should_receive('exists').and_return(True)
flexmock(module).should_receive('any_parent_directories').replace_with(
lambda path, candidates: any(path.startswith(parent) for parent in candidates)
)
flexmock(module).should_receive('special_file').with_args('/foo', None).and_return(False)
flexmock(module).should_receive('special_file').with_args('/bar', None).and_return(False)
flexmock(module).should_receive('special_file').with_args('/baz', None).and_return(False)
flexmock(module.logger).should_receive('warning').once()
assert module.validate_planned_backup_paths(
dry_run=False,
create_command=('borg', 'create'),
config={},
patterns=(
module.borgmatic.borg.pattern.Pattern('/foo'),
module.borgmatic.borg.pattern.Pattern(
'/run/borgmatic/bar', module.borgmatic.borg.pattern.Pattern_type.ROOT
assert (
module.validate_planned_backup_paths(
dry_run=False,
create_command=('borg', 'create'),
config={},
patterns=(
module.borgmatic.borg.pattern.Pattern('/foo'),
module.borgmatic.borg.pattern.Pattern(
'/run/borgmatic/bar', module.borgmatic.borg.pattern.Pattern_type.ROOT
),
module.borgmatic.borg.pattern.Pattern('/baz'),
),
module.borgmatic.borg.pattern.Pattern('/baz'),
),
local_path=None,
working_directory=None,
borgmatic_runtime_directory='/run/borgmatic',
) == ('/foo', '/bar', '/baz')
local_path=None,
working_directory=None,
borgmatic_runtime_directory='/run/borgmatic',
)
== ()
)
def test_validate_planned_backup_paths_with_borgmatic_runtime_directory_partially_excluded_from_paths_output_does_not_warn():
@@ -165,33 +192,42 @@ def test_validate_planned_backup_paths_with_borgmatic_runtime_directory_partiall
flexmock(module.environment).should_receive('make_environment').and_return(None)
# /run/borgmatic/bar is present, but /run/borgmatic/quux is missing.
flexmock(module).should_receive('execute_command_and_capture_output').and_return(
'+ /foo\n- /run/borgmatic/bar\n- /baz',
flexmock(module).should_receive('execute_command_and_capture_output').and_yield(
'+ /foo',
'- /run/borgmatic/bar',
'- /baz',
)
flexmock(module.os.path).should_receive('exists').and_return(True)
flexmock(module).should_receive('any_parent_directories').replace_with(
lambda path, candidates: any(path.startswith(parent) for parent in candidates)
)
flexmock(module).should_receive('special_file').with_args('/foo', None).and_return(False)
flexmock(module).should_receive('special_file').with_args('/run/borgmatic/bar', None).never()
flexmock(module).should_receive('special_file').with_args('/baz', None).and_return(False)
flexmock(module).should_receive('special_file').with_args('/run/borgmatic/quux', None).never()
flexmock(module.logger).should_receive('warning').never()
assert module.validate_planned_backup_paths(
dry_run=False,
create_command=('borg', 'create'),
config={},
patterns=(
module.borgmatic.borg.pattern.Pattern('/foo'),
module.borgmatic.borg.pattern.Pattern(
'/run/borgmatic/bar', module.borgmatic.borg.pattern.Pattern_type.ROOT
assert (
module.validate_planned_backup_paths(
dry_run=False,
create_command=('borg', 'create'),
config={},
patterns=(
module.borgmatic.borg.pattern.Pattern('/foo'),
module.borgmatic.borg.pattern.Pattern(
'/run/borgmatic/bar', module.borgmatic.borg.pattern.Pattern_type.ROOT
),
module.borgmatic.borg.pattern.Pattern('/baz'),
module.borgmatic.borg.pattern.Pattern(
'/run/borgmatic/quux', module.borgmatic.borg.pattern.Pattern_type.ROOT
),
),
module.borgmatic.borg.pattern.Pattern('/baz'),
module.borgmatic.borg.pattern.Pattern(
'/run/borgmatic/quux', module.borgmatic.borg.pattern.Pattern_type.ROOT
),
),
local_path=None,
working_directory=None,
borgmatic_runtime_directory='/run/borgmatic',
) == ('/foo', '/baz')
local_path=None,
working_directory=None,
borgmatic_runtime_directory='/run/borgmatic',
)
== ()
)
def test_validate_planned_backup_paths_with_borgmatic_runtime_directory_with_corresponding_include_and_missing_from_paths_output_does_not_warn():
@@ -202,33 +238,41 @@ def test_validate_planned_backup_paths_with_borgmatic_runtime_directory_with_cor
lambda arguments, flag: arguments,
)
flexmock(module.environment).should_receive('make_environment').and_return(None)
flexmock(module).should_receive('execute_command_and_capture_output').and_return(
'+ /foo\n- /bar\n- /baz',
flexmock(module).should_receive('execute_command_and_capture_output').and_yield(
'+ /foo',
'- /bar',
'- /baz',
)
flexmock(module.os.path).should_receive('exists').and_return(True)
flexmock(module).should_receive('any_parent_directories').replace_with(
lambda path, candidates: any(path.startswith(parent) for parent in candidates)
)
flexmock(module).should_receive('special_file').with_args('/foo', None).and_return(False)
flexmock(module).should_receive('special_file').with_args('/bar', None).and_return(False)
flexmock(module).should_receive('special_file').with_args('/baz', None).and_return(False)
flexmock(module.logger).should_receive('warning').never()
assert module.validate_planned_backup_paths(
dry_run=False,
create_command=('borg', 'create'),
config={},
patterns=(
module.borgmatic.borg.pattern.Pattern('/foo'),
module.borgmatic.borg.pattern.Pattern(
'/run/borgmatic/bar', module.borgmatic.borg.pattern.Pattern_type.ROOT
assert (
module.validate_planned_backup_paths(
dry_run=False,
create_command=('borg', 'create'),
config={},
patterns=(
module.borgmatic.borg.pattern.Pattern('/foo'),
module.borgmatic.borg.pattern.Pattern(
'/run/borgmatic/bar', module.borgmatic.borg.pattern.Pattern_type.ROOT
),
module.borgmatic.borg.pattern.Pattern(
'/run/borgmatic/bar', module.borgmatic.borg.pattern.Pattern_type.INCLUDE
),
module.borgmatic.borg.pattern.Pattern('/baz'),
),
module.borgmatic.borg.pattern.Pattern(
'/run/borgmatic/bar', module.borgmatic.borg.pattern.Pattern_type.INCLUDE
),
module.borgmatic.borg.pattern.Pattern('/baz'),
),
local_path=None,
working_directory=None,
borgmatic_runtime_directory='/run/borgmatic',
) == ('/foo', '/bar', '/baz')
local_path=None,
working_directory=None,
borgmatic_runtime_directory='/run/borgmatic',
)
== ()
)
def test_validate_planned_backup_paths_with_borgmatic_runtime_directory_missing_from_patterns_does_not_raise():
@@ -239,26 +283,33 @@ def test_validate_planned_backup_paths_with_borgmatic_runtime_directory_missing_
lambda arguments, flag: arguments,
)
flexmock(module.environment).should_receive('make_environment').and_return(None)
flexmock(module).should_receive('execute_command_and_capture_output').and_return(
'+ /foo\n- /run/borgmatic/bar\n- /baz',
flexmock(module).should_receive('execute_command_and_capture_output').and_yield(
'+ /foo',
'- /run/borgmatic/bar',
'- /baz',
)
flexmock(module.os.path).should_receive('exists').and_return(True)
flexmock(module).should_receive('any_parent_directories').replace_with(
lambda path, candidates: any(path.startswith(parent) for parent in candidates)
)
flexmock(module).should_receive('special_file').with_args('/foo', None).and_return(False)
flexmock(module).should_receive('special_file').with_args('/baz', None).and_return(False)
assert module.validate_planned_backup_paths(
dry_run=False,
create_command=('borg', 'create'),
config={},
patterns=(
module.borgmatic.borg.pattern.Pattern('/foo'),
module.borgmatic.borg.pattern.Pattern('/baz'),
),
local_path=None,
working_directory=None,
borgmatic_runtime_directory='/run/borgmatic',
) == ('/foo', '/baz')
assert (
module.validate_planned_backup_paths(
dry_run=False,
create_command=('borg', 'create'),
config={},
patterns=(
module.borgmatic.borg.pattern.Pattern('/foo'),
module.borgmatic.borg.pattern.Pattern('/baz'),
),
local_path=None,
working_directory=None,
borgmatic_runtime_directory='/run/borgmatic',
)
== ()
)
def test_validate_planned_backup_paths_with_dry_run_and_borgmatic_runtime_directory_missing_from_paths_output_does_not_raise():
@@ -269,27 +320,37 @@ def test_validate_planned_backup_paths_with_dry_run_and_borgmatic_runtime_direct
lambda arguments, flag: arguments,
)
flexmock(module.environment).should_receive('make_environment').and_return(None)
flexmock(module).should_receive('execute_command_and_capture_output').and_return(
'+ /foo\n- /run/borgmatic/bar\n- /baz',
flexmock(module).should_receive('execute_command_and_capture_output').and_yield(
'+ /foo',
'- /run/borgmatic/bar',
'- /baz',
)
flexmock(module.os.path).should_receive('exists').and_return(True)
flexmock(module).should_receive('any_parent_directories').and_return(False)
flexmock(module).should_receive('any_parent_directories').replace_with(
lambda path, candidates: any(path.startswith(parent) for parent in candidates)
)
flexmock(module).should_receive('special_file').with_args('/foo', None).and_return(False)
flexmock(module).should_receive('special_file').with_args('/run/borgmatic/bar', None).never()
flexmock(module).should_receive('special_file').with_args('/baz', None).and_return(False)
assert module.validate_planned_backup_paths(
dry_run=True,
create_command=('borg', 'create'),
config={},
patterns=(
module.borgmatic.borg.pattern.Pattern('/foo'),
module.borgmatic.borg.pattern.Pattern(
'/run/borgmatic/bar', module.borgmatic.borg.pattern.Pattern_type.ROOT
assert (
module.validate_planned_backup_paths(
dry_run=True,
create_command=('borg', 'create'),
config={},
patterns=(
module.borgmatic.borg.pattern.Pattern('/foo'),
module.borgmatic.borg.pattern.Pattern(
'/run/borgmatic/bar', module.borgmatic.borg.pattern.Pattern_type.ROOT
),
module.borgmatic.borg.pattern.Pattern('/baz'),
),
module.borgmatic.borg.pattern.Pattern('/baz'),
),
local_path=None,
working_directory=None,
borgmatic_runtime_directory='/run/borgmatic',
) == ('/foo', '/run/borgmatic/bar', '/baz')
local_path=None,
working_directory=None,
borgmatic_runtime_directory='/run/borgmatic',
)
== ()
)
DEFAULT_ARCHIVE_NAME = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
@@ -718,18 +779,7 @@ def test_make_base_create_command_with_stream_processes_ignores_read_special_fal
)
flexmock(module.logger).should_receive('warning').twice()
flexmock(module.environment).should_receive('make_environment')
flexmock(module).should_receive('validate_planned_backup_paths').and_return(
(
'/non/special',
'/dev/null',
)
)
flexmock(module).should_receive('special_file').with_args(
'/non/special', working_directory=None
).and_return(False)
flexmock(module).should_receive('special_file').with_args(
'/dev/null', working_directory=None
).and_return(True)
flexmock(module).should_receive('validate_planned_backup_paths').and_return(('/dev/null',))
flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').with_args(
(
Pattern(
@@ -788,18 +838,7 @@ def test_make_base_create_command_without_patterns_and_with_stream_processes_ign
)
flexmock(module.logger).should_receive('warning').twice()
flexmock(module.environment).should_receive('make_environment')
flexmock(module).should_receive('validate_planned_backup_paths').and_return(
(
'/non/special',
'/dev/null',
)
)
flexmock(module).should_receive('special_file').with_args(
'/non/special', working_directory=None
).and_return(False)
flexmock(module).should_receive('special_file').with_args(
'/dev/null', working_directory=None
).and_return(True)
flexmock(module).should_receive('validate_planned_backup_paths').and_return(('/dev/null',))
flexmock(module.borgmatic.borg.pattern).should_receive('write_patterns_file').with_args(
(
Pattern(
@@ -1271,7 +1310,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
environment=None,
borg_local_path='borg',
borg_exit_codes=None,
)
).and_yield()
insert_logging_mock(logging.INFO)
module.create_archive(
@@ -1338,7 +1377,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
environment=None,
borg_local_path='borg',
borg_exit_codes=None,
)
).and_yield()
insert_logging_mock(logging.DEBUG)
module.create_archive(
@@ -1628,7 +1667,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr
borg_exit_codes=None,
working_directory=None,
environment=None,
)
).and_yield()
flexmock(module).should_receive('execute_command_with_processes').with_args(
create_command,
processes=processes,
@@ -1638,7 +1677,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr
borg_exit_codes=None,
working_directory=None,
environment=None,
)
).and_yield()
module.create_archive(
dry_run=False,
@@ -1671,7 +1710,7 @@ def test_create_archive_with_json_calls_borg_with_json_flag():
environment=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return('[]')
).and_yield('[]')
json_output = module.create_archive(
dry_run=False,
@@ -1705,7 +1744,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_flag():
environment=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return('[]')
).and_yield('[]')
json_output = module.create_archive(
dry_run=False,
@@ -1794,3 +1833,85 @@ def test_create_archive_calls_borg_with_working_directory():
global_arguments=flexmock(),
borgmatic_runtime_directory='/borgmatic/run',
)
def test_validate_planned_backup_paths_returns_special_files():
flexmock(module.flags).should_receive('omit_flag').replace_with(
lambda arguments, flag: arguments,
)
flexmock(module.flags).should_receive('omit_flag_and_value').replace_with(
lambda arguments, flag: arguments,
)
flexmock(module.environment).should_receive('make_environment').and_return(None)
flexmock(module).should_receive('execute_command_and_capture_output').and_yield(
'+ /dev/foo',
'- /run/borgmatic/bar',
'- /dev/baz',
'- /quux',
)
flexmock(module.os.path).should_receive('exists').and_return(True)
flexmock(module).should_receive('any_parent_directories').replace_with(
lambda path, candidates: any(path.startswith(parent) for parent in candidates)
)
flexmock(module).should_receive('special_file').with_args('/dev/foo', None).and_return(True)
flexmock(module).should_receive('special_file').with_args('/run/borgmatic/bar', None).never()
flexmock(module).should_receive('special_file').with_args('/dev/baz', None).and_return(True)
flexmock(module).should_receive('special_file').with_args('/quux', None).and_return(False)
assert module.validate_planned_backup_paths(
dry_run=False,
create_command=('borg', 'create'),
config={},
patterns=(
module.borgmatic.borg.pattern.Pattern('/dev/foo'),
module.borgmatic.borg.pattern.Pattern(
'/run/borgmatic/bar', module.borgmatic.borg.pattern.Pattern_type.ROOT
),
module.borgmatic.borg.pattern.Pattern('/dev/baz'),
),
local_path=None,
working_directory=None,
borgmatic_runtime_directory='/run/borgmatic',
find_special_files=True,
) == ('/dev/foo', '/dev/baz')
def test_validate_planned_backup_paths_without_find_special_files_ignores_special_files():
flexmock(module.flags).should_receive('omit_flag').replace_with(
lambda arguments, flag: arguments,
)
flexmock(module.flags).should_receive('omit_flag_and_value').replace_with(
lambda arguments, flag: arguments,
)
flexmock(module.environment).should_receive('make_environment').and_return(None)
flexmock(module).should_receive('execute_command_and_capture_output').and_yield(
'+ /dev/foo',
'- /run/borgmatic/bar',
'- /dev/baz',
'- /quux',
)
flexmock(module.os.path).should_receive('exists').and_return(True)
flexmock(module).should_receive('any_parent_directories').replace_with(
lambda path, candidates: any(path.startswith(parent) for parent in candidates)
)
flexmock(module).should_receive('special_file').never()
assert (
module.validate_planned_backup_paths(
dry_run=False,
create_command=('borg', 'create'),
config={},
patterns=(
module.borgmatic.borg.pattern.Pattern('/dev/foo'),
module.borgmatic.borg.pattern.Pattern(
'/run/borgmatic/bar', module.borgmatic.borg.pattern.Pattern_type.ROOT
),
module.borgmatic.borg.pattern.Pattern('/dev/baz'),
),
local_path=None,
working_directory=None,
borgmatic_runtime_directory='/run/borgmatic',
find_special_files=False,
)
== ()
)
tests/unit/borg/test_info.py
+4 -5
View File
@@ -571,7 +571,7 @@ def test_display_archives_info_calls_two_commands():
flexmock(module).should_receive('make_info_command')
flexmock(module.environment).should_receive('make_environment')
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
flexmock(module).should_receive('execute_command_and_capture_output').once()
flexmock(module).should_receive('execute_command_and_capture_output').and_yield().once()
flexmock(module.flags).should_receive('warn_for_aggressive_archive_flags')
flexmock(module).should_receive('execute_command').once()
@@ -590,8 +590,7 @@ def test_display_archives_info_with_json_calls_json_command_only():
flexmock(module).should_receive('make_info_command')
flexmock(module.environment).should_receive('make_environment')
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
json_output = flexmock()
flexmock(module).should_receive('execute_command_and_capture_output').and_return(json_output)
flexmock(module).should_receive('execute_command_and_capture_output').and_yield('{}')
flexmock(module.flags).should_receive('warn_for_aggressive_archive_flags').never()
flexmock(module).should_receive('execute_command').never()
@@ -603,7 +602,7 @@ def test_display_archives_info_with_json_calls_json_command_only():
global_arguments=flexmock(),
info_arguments=flexmock(archive=None, json=True, prefix=None, match_archives=None),
)
== json_output
== '{}'
)
@@ -621,7 +620,7 @@ def test_display_archives_info_calls_borg_with_working_directory():
working_directory='/working/dir',
borg_local_path=object,
borg_exit_codes=object,
).once()
).and_yield().once()
flexmock(module.flags).should_receive('warn_for_aggressive_archive_flags')
flexmock(module).should_receive('execute_command').with_args(
full_command=object,
tests/unit/borg/test_list.py
+3 -3
View File
@@ -332,7 +332,7 @@ def test_make_find_paths_adds_globs_to_path_fragments():
def test_capture_archive_listing_does_not_raise():
flexmock(module.environment).should_receive('make_environment')
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
flexmock(module).should_receive('execute_command_and_capture_output').and_return('')
flexmock(module).should_receive('execute_command_and_capture_output').and_yield('')
flexmock(module).should_receive('make_list_command')
module.capture_archive_listing(
@@ -537,7 +537,7 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths():
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return('archive1\narchive2').once()
).and_yield('archive1', 'archive2').once()
flexmock(module).should_receive('make_list_command').and_return(
('borg', 'list', '--log-json', 'repo::archive1'),
).and_return(('borg', 'list', '--log-json', 'repo::archive2'))
@@ -802,7 +802,7 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return('archive1\narchive2').once()
).and_yield('archive1', 'archive2').once()
flexmock(module).should_receive('make_list_command').with_args(
repository_path='repo',
tests/unit/borg/test_passcommand.py
+2 -2
View File
@@ -7,7 +7,7 @@ def test_run_passcommand_does_not_raise():
module.run_passcommand.cache_clear()
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return('passphrase')
).and_yield('passphrase')
assert module.run_passcommand('passcommand', working_directory=None) == 'passphrase'
@@ -39,7 +39,7 @@ def test_run_passcommand_caches_passcommand_after_first_call():
module.run_passcommand.cache_clear()
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return('passphrase').once()
).and_yield('passphrase').once()
assert module.run_passcommand('passcommand', working_directory=None) == 'passphrase'
assert module.run_passcommand('passcommand', working_directory=None) == 'passphrase'
tests/unit/borg/test_repo_info.py
+14 -14
View File
@@ -28,7 +28,7 @@ def test_display_repository_info_calls_borg_with_flags():
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return('[]')
).and_yield('[]')
flexmock(module.flags).should_receive('warn_for_aggressive_archive_flags')
flexmock(module).should_receive('execute_command').with_args(
('borg', 'repo-info', '--log-json', '--repo', 'repo'),
@@ -64,7 +64,7 @@ def test_display_repository_info_without_borg_features_calls_borg_with_info_sub_
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return('[]')
).and_yield('[]')
flexmock(module.flags).should_receive('warn_for_aggressive_archive_flags')
flexmock(module).should_receive('execute_command').with_args(
('borg', 'info', '--log-json', 'repo'),
@@ -105,7 +105,7 @@ def test_display_repository_info_with_log_info_calls_borg_with_info_flag():
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return('[]')
).and_yield('[]')
flexmock(module.flags).should_receive('warn_for_aggressive_archive_flags')
flexmock(module).should_receive('execute_command').with_args(
('borg', 'repo-info', '--info', '--log-json', '--repo', 'repo'),
@@ -146,7 +146,7 @@ def test_display_repository_info_with_log_info_and_json_suppresses_most_borg_out
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return('[]')
).and_yield('[]')
flexmock(module.flags).should_receive('warn_for_aggressive_archive_flags').never()
insert_logging_mock(logging.INFO)
@@ -182,7 +182,7 @@ def test_display_repository_info_with_log_debug_calls_borg_with_debug_flag():
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return('[]')
).and_yield('[]')
flexmock(module.flags).should_receive('warn_for_aggressive_archive_flags')
flexmock(module).should_receive('execute_command').with_args(
('borg', 'repo-info', '--debug', '--show-rc', '--log-json', '--repo', 'repo'),
@@ -224,7 +224,7 @@ def test_display_repository_info_with_log_debug_and_json_suppresses_most_borg_ou
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return('[]')
).and_yield('[]')
flexmock(module.flags).should_receive('warn_for_aggressive_archive_flags').never()
insert_logging_mock(logging.DEBUG)
@@ -260,7 +260,7 @@ def test_display_repository_info_with_json_calls_borg_with_json_flag():
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return('[]')
).and_yield('[]')
flexmock(module.flags).should_receive('warn_for_aggressive_archive_flags').never()
json_output = module.display_repository_info(
@@ -295,7 +295,7 @@ def test_display_repository_info_with_local_path_calls_borg_via_local_path():
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return('[]')
).and_yield('[]')
flexmock(module.flags).should_receive('warn_for_aggressive_archive_flags')
flexmock(module).should_receive('execute_command').with_args(
('borg1', 'repo-info', '--log-json', '--repo', 'repo'),
@@ -338,7 +338,7 @@ def test_display_repository_info_with_exit_codes_calls_borg_using_them():
working_directory=None,
borg_local_path='borg',
borg_exit_codes=borg_exit_codes,
).and_return('[]')
).and_yield('[]')
flexmock(module.flags).should_receive('warn_for_aggressive_archive_flags')
flexmock(module).should_receive('execute_command').with_args(
('borg', 'repo-info', '--log-json', '--repo', 'repo'),
@@ -379,7 +379,7 @@ def test_display_repository_info_with_remote_path_calls_borg_with_remote_path_fl
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return('[]')
).and_yield('[]')
flexmock(module.flags).should_receive('warn_for_aggressive_archive_flags')
flexmock(module).should_receive('execute_command').with_args(
('borg', 'repo-info', '--remote-path', 'borg1', '--log-json', '--repo', 'repo'),
@@ -421,7 +421,7 @@ def test_display_repository_info_with_umask_calls_borg_with_umask_flags():
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return('[]')
).and_yield('[]')
flexmock(module.flags).should_receive('warn_for_aggressive_archive_flags')
flexmock(module).should_receive('execute_command').with_args(
('borg', 'repo-info', '--umask', '077', '--log-json', '--repo', 'repo'),
@@ -499,7 +499,7 @@ def test_display_repository_info_without_feature_available_calls_borg_with_info_
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return('[]')
).and_yield('[]')
flexmock(module.flags).should_receive('warn_for_aggressive_archive_flags')
flexmock(module).should_receive('execute_command').with_args(
('borg', 'info', '--log-json', '--extra', 'value with space', '--repo', 'repo'),
@@ -550,7 +550,7 @@ def test_display_repository_info_with_feature_available_calls_borg_with_repo_inf
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return('[]')
).and_yield('[]')
flexmock(module.flags).should_receive('warn_for_aggressive_archive_flags')
flexmock(module).should_receive('execute_command').with_args(
('borg', 'repo-info', '--log-json', '--extra', 'value with space', '--repo', 'repo'),
@@ -593,7 +593,7 @@ def test_display_repository_info_calls_borg_with_working_directory():
working_directory='/working/dir',
borg_local_path='borg',
borg_exit_codes=None,
).and_return('[]')
).and_yield('[]')
flexmock(module.flags).should_receive('warn_for_aggressive_archive_flags')
flexmock(module).should_receive('execute_command').with_args(
('borg', 'repo-info', '--log-json', '--repo', 'repo'),
tests/unit/borg/test_repo_list.py
+18 -19
View File
@@ -124,7 +124,7 @@ def test_get_latest_archive_calls_borg_with_flags():
borg_exit_codes=None,
environment=None,
working_directory=None,
).and_return(json.dumps({'archives': [expected_archive]}))
).and_yield(json.dumps({'archives': [expected_archive]}))
assert (
module.get_latest_archive(
@@ -153,7 +153,7 @@ def test_get_latest_archive_with_log_info_calls_borg_without_info_flag():
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return(json.dumps({'archives': [expected_archive]}))
).and_yield(json.dumps({'archives': [expected_archive]}))
insert_logging_mock(logging.INFO)
assert (
@@ -183,7 +183,7 @@ def test_get_latest_archive_with_log_debug_calls_borg_without_debug_flag():
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return(json.dumps({'archives': [expected_archive]}))
).and_yield(json.dumps({'archives': [expected_archive]}))
insert_logging_mock(logging.DEBUG)
assert (
@@ -213,7 +213,7 @@ def test_get_latest_archive_with_local_path_calls_borg_via_local_path():
working_directory=None,
borg_local_path='borg1',
borg_exit_codes=None,
).and_return(json.dumps({'archives': [expected_archive]}))
).and_yield(json.dumps({'archives': [expected_archive]}))
assert (
module.get_latest_archive(
@@ -244,7 +244,7 @@ def test_get_latest_archive_with_exit_codes_calls_borg_using_them():
working_directory=None,
borg_local_path='borg',
borg_exit_codes=borg_exit_codes,
).and_return(json.dumps({'archives': [expected_archive]}))
).and_yield(json.dumps({'archives': [expected_archive]}))
assert (
module.get_latest_archive(
@@ -276,7 +276,7 @@ def test_get_latest_archive_with_remote_path_calls_borg_with_remote_path_flags()
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return(json.dumps({'archives': [expected_archive]}))
).and_yield(json.dumps({'archives': [expected_archive]}))
assert (
module.get_latest_archive(
@@ -309,7 +309,7 @@ def test_get_latest_archive_with_umask_calls_borg_with_umask_flags():
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return(json.dumps({'archives': [expected_archive]}))
).and_yield(json.dumps({'archives': [expected_archive]}))
assert (
module.get_latest_archive(
@@ -337,7 +337,7 @@ def test_get_latest_archive_without_archives_raises():
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return(json.dumps({'archives': []}))
).and_yield(json.dumps({'archives': []}))
with pytest.raises(ValueError):
module.get_latest_archive(
@@ -367,7 +367,7 @@ def test_get_latest_archive_with_lock_wait_calls_borg_with_lock_wait_flags():
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return(json.dumps({'archives': [expected_archive]}))
).and_yield(json.dumps({'archives': [expected_archive]}))
assert (
module.get_latest_archive(
@@ -404,7 +404,7 @@ def test_get_latest_archive_calls_borg_with_list_extra_borg_options():
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return(json.dumps({'archives': [expected_archive]}))
).and_yield(json.dumps({'archives': [expected_archive]}))
assert (
module.get_latest_archive(
@@ -441,7 +441,7 @@ def test_get_latest_archive_with_feature_available_calls_borg_with_repo_list_ext
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return(json.dumps({'archives': [expected_archive]}))
).and_yield(json.dumps({'archives': [expected_archive]}))
assert (
module.get_latest_archive(
@@ -473,7 +473,7 @@ def test_get_latest_archive_with_consider_checkpoints_calls_borg_with_consider_c
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return(json.dumps({'archives': [expected_archive]}))
).and_yield(json.dumps({'archives': [expected_archive]}))
assert (
module.get_latest_archive(
@@ -506,7 +506,7 @@ def test_get_latest_archive_with_consider_checkpoints_and_feature_available_call
working_directory=None,
borg_local_path='borg',
borg_exit_codes=None,
).and_return(json.dumps({'archives': [expected_archive]}))
).and_yield(json.dumps({'archives': [expected_archive]}))
assert (
module.get_latest_archive(
@@ -538,7 +538,7 @@ def test_get_latest_archive_calls_borg_with_working_directory():
borg_exit_codes=None,
environment=None,
working_directory='/working/dir',
).and_return(json.dumps({'archives': [expected_archive]}))
).and_yield(json.dumps({'archives': [expected_archive]}))
assert (
module.get_latest_archive(
@@ -1114,7 +1114,7 @@ def test_list_repository_calls_two_commands():
flexmock(module).should_receive('make_repo_list_command')
flexmock(module.environment).should_receive('make_environment')
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
flexmock(module).should_receive('execute_command_and_capture_output').once()
flexmock(module).should_receive('execute_command_and_capture_output').and_yield('').once()
flexmock(module.flags).should_receive('warn_for_aggressive_archive_flags')
flexmock(module).should_receive('execute_command').once()
@@ -1132,8 +1132,7 @@ def test_list_repository_with_json_calls_json_command_only():
flexmock(module).should_receive('make_repo_list_command')
flexmock(module.environment).should_receive('make_environment')
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(None)
json_output = flexmock()
flexmock(module).should_receive('execute_command_and_capture_output').and_return(json_output)
flexmock(module).should_receive('execute_command_and_capture_output').and_yield('{}')
flexmock(module.flags).should_receive('warn_for_aggressive_archive_flags').never()
flexmock(module).should_receive('execute_command').never()
@@ -1145,7 +1144,7 @@ def test_list_repository_with_json_calls_json_command_only():
repo_list_arguments=argparse.Namespace(json=True),
global_arguments=flexmock(),
)
== json_output
== '{}'
)
@@ -1211,7 +1210,7 @@ def test_list_repository_calls_borg_with_working_directory():
working_directory='/working/dir',
borg_local_path=object,
borg_exit_codes=object,
).once()
).and_yield('').once()
flexmock(module.flags).should_receive('warn_for_aggressive_archive_flags')
flexmock(module).should_receive('execute_command').with_args(
full_command=object,
tests/unit/borg/test_version.py
+1 -1
View File
@@ -27,7 +27,7 @@ def insert_execute_command_and_capture_output_mock(
working_directory=working_directory,
borg_local_path=borg_local_path,
borg_exit_codes=borg_exit_codes,
).once().and_return(version_output)
).and_yield(version_output).once()
def test_local_borg_version_calls_borg_with_required_parameters():
tests/unit/hooks/credential/test_keepassxc.py
+7 -7
View File
@@ -48,7 +48,7 @@ def test_load_credential_with_present_database_fetches_password_from_keepassxc()
'database.kdbx',
'mypassword',
),
).and_return('password').once()
).and_yield('password').once()
assert (
module.load_credential(
@@ -79,7 +79,7 @@ def test_load_credential_with_custom_keepassxc_cli_command_calls_it():
'database.kdbx',
'mypassword',
),
).and_return('password').once()
).and_yield('password').once()
assert (
module.load_credential(
@@ -108,7 +108,7 @@ def test_load_credential_with_expanded_directory_with_present_database_fetches_p
'/root/database.kdbx',
'mypassword',
),
).and_return('password').once()
).and_yield('password').once()
assert (
module.load_credential(
@@ -139,7 +139,7 @@ def test_load_credential_with_key_file():
'database.kdbx',
'mypassword',
),
).and_return('password').once()
).and_yield('password').once()
assert (
module.load_credential(
@@ -171,7 +171,7 @@ def test_load_credential_with_key_file_and_ask_for_password_false():
'database.kdbx',
'mypassword',
),
).and_return('password').once()
).and_yield('password').once()
assert (
module.load_credential(
@@ -202,7 +202,7 @@ def test_load_credential_with_yubikey():
'database.kdbx',
'mypassword',
),
).and_return('password').once()
).and_yield('password').once()
assert (
module.load_credential(
@@ -235,7 +235,7 @@ def test_load_credential_with_key_file_and_yubikey():
'database.kdbx',
'mypassword',
),
).and_return('password').once()
).and_yield('password').once()
assert (
module.load_credential(
tests/unit/hooks/credential/test_systemd.py
+3 -3
View File
@@ -25,7 +25,7 @@ def test_load_credential_without_credentials_directory_falls_back_to_systemd_cre
)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).with_args(('systemd-creds', 'decrypt', '/etc/credstore.encrypted/mycredential')).and_return(
).with_args(('systemd-creds', 'decrypt', '/etc/credstore.encrypted/mycredential')).and_yield(
'password'
).once()
@@ -43,7 +43,7 @@ def test_load_credential_without_credentials_directory_calls_custom_systemd_cred
'execute_command_and_capture_output'
).with_args(
('/path/to/systemd-creds', '--flag', 'decrypt', '/etc/credstore.encrypted/mycredential')
).and_return('password').once()
).and_yield('password').once()
assert (
module.load_credential(
@@ -61,7 +61,7 @@ def test_load_credential_without_credentials_directory_uses_custom_encrypted_cre
)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).with_args(('systemd-creds', 'decrypt', '/my/credstore.encrypted/mycredential')).and_return(
).with_args(('systemd-creds', 'decrypt', '/my/credstore.encrypted/mycredential')).and_yield(
'password'
).once()
tests/unit/hooks/data_source/test_btrfs.py
+5 -5
View File
@@ -42,7 +42,7 @@ def test_get_subvolume_property_with_invalid_btrfs_output_errors():
module.get_subvolume_property.cache_clear()
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return('invalid')
).and_yield('invalid')
with pytest.raises(ValueError):
module.get_subvolume_property('btrfs', '/foo', 'ro')
@@ -52,7 +52,7 @@ def test_get_subvolume_property_with_true_output_returns_true_bool():
module.get_subvolume_property.cache_clear()
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return('ro=true')
).and_yield('ro=true')
assert module.get_subvolume_property('btrfs', '/foo', 'ro') is True
@@ -61,7 +61,7 @@ def test_get_subvolume_property_with_false_output_returns_false_bool():
module.get_subvolume_property.cache_clear()
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return('ro=false')
).and_yield('ro=false')
assert module.get_subvolume_property('btrfs', '/foo', 'ro') is False
@@ -70,7 +70,7 @@ def test_get_subvolume_property_passes_through_general_value():
module.get_subvolume_property.cache_clear()
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return('thing=value')
).and_yield('thing=value')
assert module.get_subvolume_property('btrfs', '/foo', 'thing') == 'value'
@@ -79,7 +79,7 @@ def test_get_subvolume_property_caches_result_after_first_call():
module.get_subvolume_property.cache_clear()
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return('thing=value').once()
).and_yield('thing=value').once()
assert module.get_subvolume_property('btrfs', '/foo', 'thing') == 'value'
assert module.get_subvolume_property('btrfs', '/foo', 'thing') == 'value'
tests/unit/hooks/data_source/test_config.py
+4 -4
View File
@@ -108,7 +108,7 @@ def test_get_ip_from_container_without_engines_errors():
def test_get_ip_from_container_parses_top_level_ip_address():
flexmock(module.shutil).should_receive('which').and_return(None).and_return('/usr/bin/podman')
flexmock(module).should_receive('execute_command_and_capture_output').and_return(
flexmock(module).should_receive('execute_command_and_capture_output').and_yield(
'{"IPAddress": "1.2.3.4"}'
)
@@ -118,7 +118,7 @@ def test_get_ip_from_container_parses_top_level_ip_address():
def test_get_ip_from_container_parses_network_ip_address():
flexmock(module.shutil).should_receive('which').and_return(None).and_return('/usr/bin/podman')
flexmock(module).should_receive('execute_command_and_capture_output').and_return(
flexmock(module).should_receive('execute_command_and_capture_output').and_yield(
'{"Networks": {"my_network": {"IPAddress": "5.6.7.8"}}}'
)
@@ -138,7 +138,7 @@ def test_get_ip_from_container_without_container_errors():
def test_get_ip_from_container_without_network_errors():
flexmock(module.shutil).should_receive('which').and_return(None).and_return('/usr/bin/podman')
flexmock(module).should_receive('execute_command_and_capture_output').and_return('{}')
flexmock(module).should_receive('execute_command_and_capture_output').and_yield('{}')
with pytest.raises(ValueError) as exc_info:
module.get_ip_from_container('yolo')
@@ -149,7 +149,7 @@ def test_get_ip_from_container_without_network_errors():
def test_get_ip_from_container_with_broken_output_errors():
flexmock(module.shutil).should_receive('which').and_return(None).and_return('/usr/bin/podman')
flexmock(module).should_receive('execute_command_and_capture_output').and_return('abc')
flexmock(module).should_receive('execute_command_and_capture_output').and_yield('abc')
with pytest.raises(ValueError) as exc_info:
module.get_ip_from_container('yolo')
tests/unit/hooks/data_source/test_lvm.py
+23 -23
View File
@@ -8,8 +8,8 @@ from borgmatic.hooks.data_source import lvm as module
def test_get_logical_volumes_filters_by_patterns():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return(
'''
).and_yield(
*'''
{
"blockdevices": [
{
@@ -35,7 +35,7 @@ def test_get_logical_volumes_filters_by_patterns():
}
]
}
''',
'''.splitlines(),
)
contained = {
Pattern('/mnt/lvolume', source=Pattern_source.CONFIG),
@@ -81,8 +81,8 @@ def test_get_logical_volumes_filters_by_patterns():
def test_get_logical_volumes_skips_non_root_patterns():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return(
'''
).and_yield(
*'''
{
"blockdevices": [
{
@@ -93,7 +93,7 @@ def test_get_logical_volumes_skips_non_root_patterns():
}
]
}
''',
'''.splitlines(),
)
contained = {
Pattern('/mnt/lvolume', type=Pattern_type.EXCLUDE, source=Pattern_source.CONFIG),
@@ -130,8 +130,8 @@ def test_get_logical_volumes_skips_non_root_patterns():
def test_get_logical_volumes_skips_non_config_patterns():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return(
'''
).and_yield(
*'''
{
"blockdevices": [
{
@@ -142,7 +142,7 @@ def test_get_logical_volumes_skips_non_config_patterns():
}
]
}
''',
'''.splitlines(),
)
contained = {
Pattern('/mnt/lvolume', source=Pattern_source.HOOK),
@@ -175,7 +175,7 @@ def test_get_logical_volumes_skips_non_config_patterns():
def test_get_logical_volumes_with_invalid_lsblk_json_errors():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return('{')
).and_yield('{')
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns',
@@ -191,7 +191,7 @@ def test_get_logical_volumes_with_invalid_lsblk_json_errors():
def test_get_logical_volumes_with_lsblk_json_missing_keys_errors():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return('{"block_devices": [{}]}')
).and_yield('{"block_devices": [{}]}')
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns',
@@ -801,8 +801,8 @@ def test_dump_data_sources_with_missing_snapshot_errors():
def test_get_snapshots_lists_all_snapshots():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return(
'''
).and_yield(
*'''
{
"report": [
{
@@ -815,7 +815,7 @@ def test_get_snapshots_lists_all_snapshots():
"log": [
]
}
''',
'''.splitlines(),
)
assert module.get_snapshots('lvs') == (
@@ -828,7 +828,7 @@ def test_get_snapshots_with_snapshot_name_lists_just_that_snapshot():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return(
'''
*'''
{
"report": [
{
@@ -841,7 +841,7 @@ def test_get_snapshots_with_snapshot_name_lists_just_that_snapshot():
"log": [
]
}
''',
'''.splitlines(),
)
assert module.get_snapshots('lvs', snapshot_name='snap2') == (
@@ -852,7 +852,7 @@ def test_get_snapshots_with_snapshot_name_lists_just_that_snapshot():
def test_get_snapshots_with_invalid_lvs_json_errors():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return('{')
).and_yield('{')
with pytest.raises(ValueError):
assert module.get_snapshots('lvs')
@@ -861,14 +861,14 @@ def test_get_snapshots_with_invalid_lvs_json_errors():
def test_get_snapshots_with_lvs_json_missing_report_errors():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return(
'''
).and_yield(
*'''
{
"report": [],
"log": [
]
}
''',
'''.splitlines(),
)
with pytest.raises(ValueError):
@@ -878,8 +878,8 @@ def test_get_snapshots_with_lvs_json_missing_report_errors():
def test_get_snapshots_with_lvs_json_missing_keys_errors():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return(
'''
).and_yield(
*'''
{
"report": [
{
@@ -891,7 +891,7 @@ def test_get_snapshots_with_lvs_json_missing_keys_errors():
"log": [
]
}
''',
'''.splitlines(),
)
with pytest.raises(ValueError):
tests/unit/hooks/data_source/test_mariadb.py
+19 -19
View File
@@ -201,7 +201,7 @@ def test_database_names_to_dump_queries_mariadb_for_database_names():
),
environment=environment,
working_directory='/path/to/working/dir',
).and_return('foo\nbar\nmysql\n').once()
).and_yield('foo', 'bar', 'mysql').once()
names = module.database_names_to_dump(
{'name': 'all'},
@@ -239,7 +239,7 @@ def test_database_names_to_dump_with_database_name_all_and_skip_names_filters_ou
),
environment=environment,
working_directory=None,
).and_return('foo\nbar\nbaz\nmysql\n').once()
).and_yield('foo', 'bar', 'baz', 'mysql').once()
names = module.database_names_to_dump(
{'name': 'all', 'skip_names': ('foo', 'bar')},
@@ -279,7 +279,7 @@ def test_database_names_to_dump_runs_mariadb_with_socket_path():
),
environment=environment,
working_directory=None,
).and_return('foo\nbar\nmysql\n').once()
).and_yield('foo', 'bar', 'mysql').once()
names = module.database_names_to_dump(
{'name': 'all', 'socket_path': '/socket'},
@@ -314,7 +314,7 @@ def test_database_names_to_dump_with_environment_password_transport_skips_defaul
),
environment=environment,
working_directory=None,
).and_return('foo\nbar\nmysql\n').once()
).and_yield('foo', 'bar', 'mysql').once()
names = module.database_names_to_dump(
{'name': 'all', 'password_transport': 'environment'},
@@ -353,7 +353,7 @@ def test_database_names_to_dump_runs_mariadb_with_tls():
),
environment=environment,
working_directory=None,
).and_return('foo\nbar\nmysql\n').once()
).and_yield('foo', 'bar', 'mysql').once()
names = module.database_names_to_dump(
{'name': 'all', 'tls': True},
@@ -392,7 +392,7 @@ def test_database_names_to_dump_runs_mariadb_without_tls():
),
environment=environment,
working_directory=None,
).and_return('foo\nbar\nmysql\n').once()
).and_yield('foo', 'bar', 'mysql').once()
names = module.database_names_to_dump(
{'name': 'all', 'tls': False},
@@ -770,7 +770,7 @@ def test_database_names_to_dump_runs_mariadb_with_list_options():
),
environment=None,
working_directory=None,
).and_return('foo\nbar').once()
).and_yield('foo', 'bar').once()
assert module.database_names_to_dump(database, {}, 'root', 'trustsome1', None, '') == (
'foo',
@@ -807,7 +807,7 @@ def test_database_names_to_dump_runs_non_default_mariadb_with_list_options():
'show schemas',
),
working_directory=None,
).and_return('foo\nbar').once()
).and_yield('foo', 'bar').once()
assert module.database_names_to_dump(database, {}, 'root', 'trustsome1', None, '') == (
'foo',
@@ -1353,7 +1353,7 @@ def test_restore_data_source_dump_runs_mariadb_to_restore():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -1394,7 +1394,7 @@ def test_restore_data_source_dump_runs_mariadb_with_options():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -1437,7 +1437,7 @@ def test_restore_data_source_dump_runs_non_default_mariadb_with_options():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -1489,7 +1489,7 @@ def test_restore_data_source_dump_runs_mariadb_with_hostname_and_port():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -1537,7 +1537,7 @@ def test_restore_data_source_dump_runs_mariadb_with_socket_path():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -1584,7 +1584,7 @@ def test_restore_data_source_dump_runs_mariadb_with_tls():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -1631,7 +1631,7 @@ def test_restore_data_source_dump_runs_mariadb_without_tls():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -1674,7 +1674,7 @@ def test_restore_data_source_dump_runs_mariadb_with_username_and_password():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -1722,7 +1722,7 @@ def test_restore_data_source_with_environment_password_transport_skips_defaults_
input_file=extract_process.stdout,
environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'},
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -1787,7 +1787,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -1857,7 +1857,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
tests/unit/hooks/data_source/test_mongodb.py
+10 -10
View File
@@ -394,7 +394,7 @@ def test_restore_data_source_dump_runs_mongorestore():
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -438,7 +438,7 @@ def test_restore_data_source_dump_runs_mongorestore_with_hostname_and_port():
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -493,7 +493,7 @@ def test_restore_data_source_dump_runs_mongorestore_with_username_and_password()
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -556,7 +556,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -619,7 +619,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -653,7 +653,7 @@ def test_restore_data_source_dump_runs_mongorestore_with_options():
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -695,7 +695,7 @@ def test_restore_databases_dump_runs_mongorestore_with_schemas():
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -729,7 +729,7 @@ def test_restore_data_source_dump_runs_psql_for_all_database_dump():
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -788,7 +788,7 @@ def test_restore_data_source_dump_without_extract_process_restores_from_disk():
output_log_level=logging.DEBUG,
input_file=None,
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -901,7 +901,7 @@ def test_restore_data_source_dump_uses_custom_mongorestore_command():
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
tests/unit/hooks/data_source/test_mysql.py
+19 -19
View File
@@ -81,7 +81,7 @@ def test_database_names_to_dump_queries_mysql_for_database_names():
),
environment=environment,
working_directory=None,
).and_return('foo\nbar\nmysql\n').once()
).and_yield('foo', 'bar', 'mysql').once()
names = module.database_names_to_dump(
{'name': 'all'},
@@ -123,7 +123,7 @@ def test_database_names_to_dump_with_database_name_all_and_skip_names_filters_ou
),
environment=environment,
working_directory=None,
).and_return('foo\nbar\nbaz\nmysql\n').once()
).and_yield('foo', 'bar', 'baz', 'mysql').once()
names = module.database_names_to_dump(
{'name': 'all', 'skip_names': ('foo', 'bar')},
@@ -163,7 +163,7 @@ def test_database_names_to_dump_runs_mysql_with_socket_path():
),
environment=environment,
working_directory=None,
).and_return('foo\nbar\nmysql\n').once()
).and_yield('foo', 'bar', 'mysql').once()
names = module.database_names_to_dump(
{'name': 'all', 'socket_path': '/socket'},
@@ -202,7 +202,7 @@ def test_database_names_to_dump_with_environment_password_transport_skips_defaul
),
environment=environment,
working_directory=None,
).and_return('foo\nbar\nmysql\n').once()
).and_yield('foo', 'bar', 'mysql').once()
names = module.database_names_to_dump(
{'name': 'all', 'password_transport': 'environment'},
@@ -241,7 +241,7 @@ def test_database_names_to_dump_runs_mysql_with_tls():
),
environment=environment,
working_directory=None,
).and_return('foo\nbar\nmysql\n').once()
).and_yield('foo', 'bar', 'mysql').once()
names = module.database_names_to_dump(
{'name': 'all', 'tls': True},
@@ -280,7 +280,7 @@ def test_database_names_to_dump_runs_mysql_without_tls():
),
environment=environment,
working_directory=None,
).and_return('foo\nbar\nmysql\n').once()
).and_yield('foo', 'bar', 'mysql').once()
names = module.database_names_to_dump(
{'name': 'all', 'tls': False},
@@ -653,7 +653,7 @@ def test_database_names_to_dump_runs_mysql_with_list_options():
),
environment=None,
working_directory=None,
).and_return('foo\nbar').once()
).and_yield('foo', 'bar').once()
assert module.database_names_to_dump(database, {}, 'root', 'trustsome1', None, '') == (
'foo',
@@ -687,7 +687,7 @@ def test_database_names_to_dump_runs_non_default_mysql_with_list_options():
'show schemas',
),
working_directory=None,
).and_return('foo\nbar').once()
).and_yield('foo', 'bar').once()
assert module.database_names_to_dump(database, {}, 'root', 'trustsome1', None, '') == (
'foo',
@@ -1235,7 +1235,7 @@ def test_restore_data_source_dump_runs_mysql_to_restore():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -1276,7 +1276,7 @@ def test_restore_data_source_dump_runs_mysql_with_options():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -1317,7 +1317,7 @@ def test_restore_data_source_dump_runs_non_default_mysql_with_options():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -1369,7 +1369,7 @@ def test_restore_data_source_dump_runs_mysql_with_hostname_and_port():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -1421,7 +1421,7 @@ def test_restore_data_source_dump_runs_mysql_with_socket_path():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -1468,7 +1468,7 @@ def test_restore_data_source_dump_runs_mysql_with_tls():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -1515,7 +1515,7 @@ def test_restore_data_source_dump_runs_mysql_without_tls():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -1558,7 +1558,7 @@ def test_restore_data_source_dump_runs_mysql_with_username_and_password():
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -1610,7 +1610,7 @@ def test_restore_data_source_with_environment_password_transport_skips_defaults_
input_file=extract_process.stdout,
environment={'USER': 'root', 'MYSQL_PWD': 'trustsome1'},
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -1677,7 +1677,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
@@ -1749,7 +1749,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_
input_file=extract_process.stdout,
environment={'USER': 'root'},
working_directory=None,
).once()
).and_yield().once()
module.restore_data_source_dump(
hook_config,
tests/unit/hooks/data_source/test_postgresql.py
+21 -19
View File
@@ -102,8 +102,9 @@ def test_database_names_to_dump_with_all_and_format_lists_databases():
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential',
).replace_with(lambda value, config: value)
flexmock(module).should_receive('execute_command_and_capture_output').and_return(
'foo,test,\nbar,test,"stuff and such"',
flexmock(module).should_receive('execute_command_and_capture_output').and_yield(
'foo,test,',
'bar,test,"stuff and such"',
)
assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == (
@@ -135,7 +136,7 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_hostnam
),
environment=object,
working_directory='/path/to/working/dir',
).and_return('foo,test,\nbar,test,"stuff and such"')
).and_yield('foo,test,', 'bar,test,"stuff and such"')
assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == (
'foo',
@@ -162,7 +163,7 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_usernam
),
environment=object,
working_directory=None,
).and_return('foo,test,\nbar,test,"stuff and such"')
).and_yield('foo,test,', 'bar,test,"stuff and such"')
assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == (
'foo',
@@ -180,7 +181,7 @@ def test_database_names_to_dump_with_all_and_format_lists_databases_with_options
('psql', '--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only', '--harder'),
environment=object,
working_directory=None,
).and_return('foo,test,\nbar,test,"stuff and such"')
).and_yield('foo,test,', 'bar,test,"stuff and such"')
assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == (
'foo',
@@ -193,8 +194,9 @@ def test_database_names_to_dump_with_all_and_format_excludes_particular_database
flexmock(module.borgmatic.hooks.credential.parse).should_receive(
'resolve_credential',
).replace_with(lambda value, config: value)
flexmock(module).should_receive('execute_command_and_capture_output').and_return(
'foo,test,\ntemplate0,test,blah',
flexmock(module).should_receive('execute_command_and_capture_output').and_yield(
'foo,test,',
'template0,test,blah',
)
assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('foo',)
@@ -226,7 +228,7 @@ def test_database_names_to_dump_with_all_and_psql_command_uses_custom_command():
),
environment=object,
working_directory=None,
).and_return('foo,text').once()
).and_yield('foo,text').once()
assert module.database_names_to_dump(database, {}, flexmock(), dry_run=False) == ('foo',)
@@ -993,7 +995,7 @@ def test_restore_data_source_dump_runs_pg_restore():
input_file=extract_process.stdout,
environment={'PGSSLMODE': 'disable'},
working_directory=None,
).once()
).and_yield().once()
flexmock(module).should_receive('execute_command').with_args(
(
'psql',
@@ -1057,7 +1059,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_hostname_and_port():
input_file=extract_process.stdout,
environment={'PGSSLMODE': 'disable'},
working_directory=None,
).once()
).and_yield().once()
flexmock(module).should_receive('execute_command').with_args(
(
'psql',
@@ -1125,7 +1127,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_username_and_password():
input_file=extract_process.stdout,
environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
working_directory=None,
).once()
).and_yield().once()
flexmock(module).should_receive('execute_command').with_args(
(
'psql',
@@ -1206,7 +1208,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_
input_file=extract_process.stdout,
environment={'PGPASSWORD': 'clipassword', 'PGSSLMODE': 'disable'},
working_directory=None,
).once()
).and_yield().once()
flexmock(module).should_receive('execute_command').with_args(
(
'psql',
@@ -1291,7 +1293,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_
input_file=extract_process.stdout,
environment={'PGPASSWORD': 'restorepassword', 'PGSSLMODE': 'disable'},
working_directory=None,
).once()
).and_yield().once()
flexmock(module).should_receive('execute_command').with_args(
(
'psql',
@@ -1363,7 +1365,7 @@ def test_restore_data_source_dump_runs_pg_restore_with_options():
input_file=extract_process.stdout,
environment={'PGSSLMODE': 'disable'},
working_directory=None,
).once()
).and_yield().once()
flexmock(module).should_receive('execute_command').with_args(
(
'psql',
@@ -1418,7 +1420,7 @@ def test_restore_data_source_dump_runs_psql_for_all_database_dump():
input_file=extract_process.stdout,
environment={'PGSSLMODE': 'disable'},
working_directory=None,
).once()
).and_yield().once()
flexmock(module).should_receive('execute_command').with_args(
('psql', '--no-password', '--no-psqlrc', '--quiet', '--command', 'ANALYZE'),
environment={'PGSSLMODE': 'disable'},
@@ -1459,7 +1461,7 @@ def test_restore_data_source_dump_runs_psql_for_plain_database_dump():
input_file=extract_process.stdout,
environment={'PGSSLMODE': 'disable'},
working_directory=None,
).once()
).and_yield().once()
flexmock(module).should_receive('execute_command').with_args(
(
'psql',
@@ -1529,7 +1531,7 @@ def test_restore_data_source_dump_runs_non_default_pg_restore_and_psql():
input_file=extract_process.stdout,
environment={'PGSSLMODE': 'disable'},
working_directory=None,
).once()
).and_yield().once()
flexmock(module).should_receive('execute_command').with_args(
(
'docker',
@@ -1619,7 +1621,7 @@ def test_restore_data_source_dump_without_extract_process_restores_from_disk():
input_file=None,
environment={'PGSSLMODE': 'disable'},
working_directory=None,
).once()
).and_yield().once()
flexmock(module).should_receive('execute_command').with_args(
(
'psql',
@@ -1681,7 +1683,7 @@ def test_restore_data_source_dump_with_schemas_restores_schemas():
input_file=None,
environment={'PGSSLMODE': 'disable'},
working_directory=None,
).once()
).and_yield().once()
flexmock(module).should_receive('execute_command').with_args(
(
'psql',
tests/unit/hooks/data_source/test_sqlite.py
+6 -6
View File
@@ -341,7 +341,7 @@ def test_restore_data_source_dump_restores_database():
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
).once()
).and_yield().once()
flexmock(module.os).should_receive('remove').once()
@@ -379,7 +379,7 @@ def test_restore_data_source_dump_runs_non_default_sqlite_restores_database():
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
).once()
).and_yield().once()
flexmock(module.os).should_receive('remove').once()
@@ -415,7 +415,7 @@ def test_restore_data_source_dump_with_connection_params_uses_connection_params_
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
).once()
).and_yield().once()
flexmock(module.os).should_receive('remove').once()
@@ -451,7 +451,7 @@ def test_restore_data_source_dump_runs_non_default_sqlite_with_connection_params
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
).once()
).and_yield().once()
flexmock(module.os).should_receive('remove').once()
@@ -489,7 +489,7 @@ def test_restore_data_source_dump_without_connection_params_uses_restore_params_
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
).once()
).and_yield().once()
flexmock(module.os).should_receive('remove').once()
@@ -526,7 +526,7 @@ def test_restore_data_source_dump_runs_non_default_sqlite_without_connection_par
output_log_level=logging.DEBUG,
input_file=extract_process.stdout,
working_directory=None,
).once()
).and_yield().once()
flexmock(module.os).should_receive('remove').once()
tests/unit/hooks/data_source/test_zfs.py
+27 -16
View File
@@ -8,8 +8,9 @@ from borgmatic.hooks.data_source import zfs as module
def test_get_datasets_to_backup_filters_datasets_by_patterns():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return(
'dataset\t/dataset\ton\t-\nother\t/other\ton\t-',
).and_yield(
'dataset\t/dataset\ton\t-',
'other\t/other\ton\t-',
)
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns',
@@ -63,8 +64,9 @@ def test_get_datasets_to_backup_filters_datasets_by_patterns():
def test_get_datasets_to_backup_skips_non_root_patterns():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return(
'dataset\t/dataset\ton\t-\nother\t/other\ton\t-',
).and_yield(
'dataset\t/dataset\ton\t-',
'other\t/other\ton\t-',
)
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns',
@@ -109,8 +111,9 @@ def test_get_datasets_to_backup_skips_non_root_patterns():
def test_get_datasets_to_backup_skips_non_config_patterns():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return(
'dataset\t/dataset\ton\t-\nother\t/other\ton\t-',
).and_yield(
'dataset\t/dataset\ton\t-',
'other\t/other\ton\t-',
)
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns',
@@ -155,8 +158,9 @@ def test_get_datasets_to_backup_skips_non_config_patterns():
def test_get_datasets_to_backup_filters_datasets_by_user_property():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return(
'dataset\t/dataset\ton\tauto\nother\t/other\ton\t-',
).and_yield(
'dataset\t/dataset\ton\tauto',
'other\t/other\ton\t-',
)
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns',
@@ -181,8 +185,9 @@ def test_get_datasets_to_backup_filters_datasets_by_user_property():
def test_get_datasets_to_backup_filters_datasets_by_canmount_property():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return(
'dataset\t/dataset\toff\t-\nother\t/other\ton\t-',
).and_yield(
'dataset\t/dataset\toff\t-',
'other\t/other\ton\t-',
)
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns',
@@ -207,7 +212,7 @@ def test_get_datasets_to_backup_filters_datasets_by_canmount_property():
def test_get_datasets_to_backup_with_invalid_list_output_raises():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return(
).and_yield(
'dataset',
)
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
@@ -221,8 +226,10 @@ def test_get_datasets_to_backup_with_invalid_list_output_raises():
def test_get_all_dataset_mount_points_omits_none():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return(
'/dataset\nnone\n/other',
).and_yield(
'/dataset',
'none',
'/other',
)
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns',
@@ -238,7 +245,10 @@ def test_get_all_dataset_mount_points_omits_duplicates():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return(
'/dataset\n/other\n/dataset\n/other',
'/dataset',
'/other',
'/dataset',
'/other',
)
flexmock(module.borgmatic.hooks.data_source.snapshot).should_receive(
'get_contained_patterns',
@@ -506,8 +516,9 @@ def test_dump_data_sources_ignores_mismatch_between_given_patterns_and_contained
def test_get_all_snapshots_parses_list_output():
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).and_return(
'dataset1@borgmatic-1234\ndataset2@borgmatic-4567',
).and_yield(
'dataset1@borgmatic-1234',
'dataset2@borgmatic-4567',
)
assert module.get_all_snapshots('zfs') == ('dataset1@borgmatic-1234', 'dataset2@borgmatic-4567')
tests/unit/test_execute.py
+97 -87
View File
@@ -207,11 +207,14 @@ def test_parse_log_line_with_came_from_stderr_and_warning_prefix_makes_warning_r
def test_handle_log_record_under_max_line_count_appends():
last_lines = ['last']
flexmock(module.logger).should_receive('handle').once()
log_record = flexmock(levelno=module.logging.INFO, getMessage=lambda: 'line')
module.handle_log_record(
flexmock(levelno=module.logging.INFO, getMessage=lambda: 'line'),
last_lines,
captured_output=flexmock(),
assert (
module.handle_log_record(
log_record,
last_lines,
)
== log_record
)
assert last_lines == ['last', 'line']
@@ -221,30 +224,19 @@ def test_handle_log_record_over_max_line_count_trims_and_appends():
original_last_lines = [str(number) for number in range(module.ERROR_OUTPUT_MAX_LINE_COUNT)]
last_lines = list(original_last_lines)
flexmock(module.logger).should_receive('handle').once()
log_record = flexmock(levelno=module.logging.INFO, getMessage=lambda: 'line')
module.handle_log_record(
flexmock(levelno=module.logging.INFO, getMessage=lambda: 'line'),
last_lines,
captured_output=flexmock(),
assert (
module.handle_log_record(
log_record,
last_lines,
)
== log_record
)
assert last_lines == [*original_last_lines[1:], 'line']
def test_handle_log_record_with_output_log_level_none_appends_captured_output():
last_lines = ['last']
captured_output = ['captured']
flexmock(module.logger).should_receive('handle').never()
module.handle_log_record(
flexmock(levelno=None, getMessage=lambda: 'line'),
last_lines,
captured_output=captured_output,
)
assert captured_output == ['captured', 'line']
def test_mask_command_secrets_masks_password_flag_value():
assert module.mask_command_secrets(('cooldb', '--username', 'bob', '--password', 'pass')) == (
'cooldb',
@@ -326,7 +318,7 @@ def test_execute_command_calls_full_command():
close_fds=False,
).and_return(flexmock(stdout=None)).once()
flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock())
flexmock(module).should_receive('log_outputs')
flexmock(module).should_receive('log_outputs').and_yield()
output = module.execute_command(full_command)
@@ -348,7 +340,7 @@ def test_execute_command_calls_full_command_with_output_file():
close_fds=False,
).and_return(flexmock(stderr=None)).once()
flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock())
flexmock(module).should_receive('log_outputs')
flexmock(module).should_receive('log_outputs').and_yield()
output = module.execute_command(full_command, output_file=output_file)
@@ -370,7 +362,7 @@ def test_execute_command_calls_full_command_without_capturing_output():
).and_return(flexmock(wait=lambda: 0)).once()
flexmock(module).should_receive('interpret_exit_code').and_return(module.Exit_status.SUCCESS)
flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock())
flexmock(module).should_receive('log_outputs')
flexmock(module).should_receive('log_outputs').and_yield()
output = module.execute_command(full_command, output_file=module.DO_NOT_CAPTURE)
@@ -392,7 +384,7 @@ def test_execute_command_calls_full_command_with_input_file():
close_fds=False,
).and_return(flexmock(stdout=None)).once()
flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock())
flexmock(module).should_receive('log_outputs')
flexmock(module).should_receive('log_outputs').and_yield()
output = module.execute_command(full_command, input_file=input_file)
@@ -413,7 +405,7 @@ def test_execute_command_calls_full_command_with_shell():
close_fds=False,
).and_return(flexmock(stdout=None)).once()
flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock())
flexmock(module).should_receive('log_outputs')
flexmock(module).should_receive('log_outputs').and_yield()
output = module.execute_command(full_command, shell=True)
@@ -434,7 +426,7 @@ def test_execute_command_calls_full_command_with_environment():
close_fds=False,
).and_return(flexmock(stdout=None)).once()
flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock())
flexmock(module).should_receive('log_outputs')
flexmock(module).should_receive('log_outputs').and_yield()
output = module.execute_command(full_command, environment={'a': 'b'})
@@ -455,7 +447,7 @@ def test_execute_command_calls_full_command_with_working_directory():
close_fds=False,
).and_return(flexmock(stdout=None)).once()
flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock())
flexmock(module).should_receive('log_outputs')
flexmock(module).should_receive('log_outputs').and_yield()
output = module.execute_command(full_command, working_directory='/working')
@@ -477,7 +469,7 @@ def test_execute_command_without_run_to_completion_returns_process():
close_fds=False,
).and_return(process).once()
flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock())
flexmock(module).should_receive('log_outputs').and_return({process: 'out'})
flexmock(module).should_receive('log_outputs').and_yield()
assert module.execute_command(full_command, run_to_completion=False) == process
@@ -497,11 +489,11 @@ def test_execute_command_and_capture_output_returns_stdout():
close_fds=False,
).and_return(process).once()
flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock())
flexmock(module).should_receive('log_outputs').and_return({process: 'out'})
flexmock(module).should_receive('log_outputs').and_yield('out')
output = module.execute_command_and_capture_output(full_command)
output_lines = tuple(module.execute_command_and_capture_output(full_command))
assert output == 'out'
assert output_lines == ('out',)
def test_execute_command_and_capture_output_with_capture_stderr_returns_stderr():
@@ -518,11 +510,13 @@ def test_execute_command_and_capture_output_with_capture_stderr_returns_stderr()
close_fds=False,
).and_return(process).once()
flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock())
flexmock(module).should_receive('log_outputs').and_return({process: 'out'})
flexmock(module).should_receive('log_outputs').and_yield('out')
output = module.execute_command_and_capture_output(full_command, capture_stderr=True)
output_lines = tuple(
module.execute_command_and_capture_output(full_command, capture_stderr=True)
)
assert output == 'out'
assert output_lines == ('out',)
def test_execute_command_and_capture_output_returns_output_when_process_error_is_not_considered_an_error():
@@ -543,9 +537,9 @@ def test_execute_command_and_capture_output_returns_output_when_process_error_is
module.Exit_status.SUCCESS,
).once()
output = module.execute_command_and_capture_output(full_command)
output_lines = tuple(module.execute_command_and_capture_output(full_command))
assert output == '[]'
assert output_lines == ('[]',)
def test_execute_command_and_capture_output_raises_when_command_errors():
@@ -566,10 +560,10 @@ def test_execute_command_and_capture_output_raises_when_command_errors():
).once()
with pytest.raises(subprocess.CalledProcessError):
module.execute_command_and_capture_output(full_command)
tuple(module.execute_command_and_capture_output(full_command))
def test_execute_command_and_capture_output_returns_output_with_shell():
def test_execute_command_and_capture_output_with_shell_returns_output():
full_command = ['foo', 'bar']
flexmock(module).should_receive('log_command')
process = flexmock()
@@ -584,14 +578,14 @@ def test_execute_command_and_capture_output_returns_output_with_shell():
close_fds=False,
).and_return(process).once()
flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock())
flexmock(module).should_receive('log_outputs').and_return({process: 'out'})
flexmock(module).should_receive('log_outputs').and_yield('out')
output = module.execute_command_and_capture_output(full_command, shell=True)
output_lines = tuple(module.execute_command_and_capture_output(full_command, shell=True))
assert output == 'out'
assert output_lines == ('out',)
def test_execute_command_and_capture_output_returns_output_with_environment():
def test_execute_command_and_capture_output_with_enviroment_returns_output():
full_command = ['foo', 'bar']
flexmock(module).should_receive('log_command')
process = flexmock()
@@ -606,15 +600,17 @@ def test_execute_command_and_capture_output_returns_output_with_environment():
close_fds=False,
).and_return(process).once()
flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock())
flexmock(module).should_receive('log_outputs').and_return({process: 'out'})
flexmock(module).should_receive('log_outputs').and_yield('out')
output = module.execute_command_and_capture_output(
full_command,
shell=False,
environment={'a': 'b'},
output_lines = tuple(
module.execute_command_and_capture_output(
full_command,
shell=False,
environment={'a': 'b'},
)
)
assert output == 'out'
assert output_lines == ('out',)
def test_execute_command_and_capture_output_returns_output_with_working_directory():
@@ -632,15 +628,17 @@ def test_execute_command_and_capture_output_returns_output_with_working_director
close_fds=False,
).and_return(process).once()
flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock())
flexmock(module).should_receive('log_outputs').and_return({process: 'out'})
flexmock(module).should_receive('log_outputs').and_yield('out')
output = module.execute_command_and_capture_output(
full_command,
shell=False,
working_directory='/working',
output_lines = tuple(
module.execute_command_and_capture_output(
full_command,
shell=False,
working_directory='/working',
)
)
assert output == 'out'
assert output_lines == ('out',)
def test_execute_command_with_processes_calls_full_command():
@@ -658,11 +656,11 @@ def test_execute_command_with_processes_calls_full_command():
close_fds=False,
).and_return(flexmock(stdout=None)).once()
flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock())
flexmock(module).should_receive('log_outputs')
flexmock(module).should_receive('log_outputs').and_yield()
output = module.execute_command_with_processes(full_command, processes)
output_lines = tuple(module.execute_command_with_processes(full_command, processes))
assert output is None
assert output_lines == ()
def test_execute_command_with_processes_returns_output_with_output_log_level_none():
@@ -681,11 +679,13 @@ def test_execute_command_with_processes_returns_output_with_output_log_level_non
close_fds=False,
).and_return(process).once()
flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock())
flexmock(module).should_receive('log_outputs').and_return({process: 'out'})
flexmock(module).should_receive('log_outputs').and_yield('out')
output = module.execute_command_with_processes(full_command, processes, output_log_level=None)
output_lines = tuple(
module.execute_command_with_processes(full_command, processes, output_log_level=None)
)
assert output == 'out'
assert output_lines == ('out',)
def test_execute_command_with_processes_calls_full_command_with_output_file():
@@ -704,11 +704,13 @@ def test_execute_command_with_processes_calls_full_command_with_output_file():
close_fds=False,
).and_return(flexmock(stderr=None)).once()
flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock())
flexmock(module).should_receive('log_outputs')
flexmock(module).should_receive('log_outputs').and_yield()
output = module.execute_command_with_processes(full_command, processes, output_file=output_file)
output_lines = tuple(
module.execute_command_with_processes(full_command, processes, output_file=output_file)
)
assert output is None
assert output_lines == ()
def test_execute_command_with_processes_calls_full_command_without_capturing_output():
@@ -727,15 +729,17 @@ def test_execute_command_with_processes_calls_full_command_without_capturing_out
).and_return(flexmock(wait=lambda: 0)).once()
flexmock(module).should_receive('interpret_exit_code').and_return(module.Exit_status.SUCCESS)
flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock())
flexmock(module).should_receive('log_outputs')
flexmock(module).should_receive('log_outputs').and_yield()
output = module.execute_command_with_processes(
full_command,
processes,
output_file=module.DO_NOT_CAPTURE,
output_lines = tuple(
module.execute_command_with_processes(
full_command,
processes,
output_file=module.DO_NOT_CAPTURE,
)
)
assert output is None
assert output_lines == ()
def test_execute_command_with_processes_calls_full_command_with_input_file():
@@ -754,11 +758,13 @@ def test_execute_command_with_processes_calls_full_command_with_input_file():
close_fds=False,
).and_return(flexmock(stdout=None)).once()
flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock())
flexmock(module).should_receive('log_outputs')
flexmock(module).should_receive('log_outputs').and_yield()
output = module.execute_command_with_processes(full_command, processes, input_file=input_file)
output_lines = tuple(
module.execute_command_with_processes(full_command, processes, input_file=input_file)
)
assert output is None
assert output_lines == ()
def test_execute_command_with_processes_calls_full_command_with_shell():
@@ -776,11 +782,11 @@ def test_execute_command_with_processes_calls_full_command_with_shell():
close_fds=False,
).and_return(flexmock(stdout=None)).once()
flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock())
flexmock(module).should_receive('log_outputs')
flexmock(module).should_receive('log_outputs').and_yield()
output = module.execute_command_with_processes(full_command, processes, shell=True)
output_lines = tuple(module.execute_command_with_processes(full_command, processes, shell=True))
assert output is None
assert output_lines == ()
def test_execute_command_with_processes_calls_full_command_with_environment():
@@ -798,11 +804,13 @@ def test_execute_command_with_processes_calls_full_command_with_environment():
close_fds=False,
).and_return(flexmock(stdout=None)).once()
flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock())
flexmock(module).should_receive('log_outputs')
flexmock(module).should_receive('log_outputs').and_yield()
output = module.execute_command_with_processes(full_command, processes, environment={'a': 'b'})
output_lines = tuple(
module.execute_command_with_processes(full_command, processes, environment={'a': 'b'})
)
assert output is None
assert output_lines == ()
def test_execute_command_with_processes_calls_full_command_with_working_directory():
@@ -820,15 +828,17 @@ def test_execute_command_with_processes_calls_full_command_with_working_director
close_fds=False,
).and_return(flexmock(stdout=None)).once()
flexmock(module.borgmatic.logger).should_receive('Log_prefix').and_return(flexmock())
flexmock(module).should_receive('log_outputs')
flexmock(module).should_receive('log_outputs').and_yield()
output = module.execute_command_with_processes(
full_command,
processes,
working_directory='/working',
output_lines = tuple(
module.execute_command_with_processes(
full_command,
processes,
working_directory='/working',
)
)
assert output is None
assert output_lines == ()
def test_execute_command_with_processes_kills_processes_on_error():
@@ -852,4 +862,4 @@ def test_execute_command_with_processes_kills_processes_on_error():
flexmock(module).should_receive('log_outputs').never()
with pytest.raises(subprocess.CalledProcessError):
module.execute_command_with_processes(full_command, processes)
tuple(module.execute_command_with_processes(full_command, processes))