Merge branch 'main' into add-diff

This commit is contained in:
Jason
2026-02-23 22:38:17 +00:00
26 changed files with 522 additions and 113 deletions
-10
View File
@@ -1,10 +0,0 @@
## Hold up, GitHub users
Thanks for your contribution!
Unfortunately, we don't use GitHub pull requests to manage code contributions to this repository (and GitHub doesn't have any way to disable pull requests entirely). Instead, please see:
https://torsion.org/borgmatic/#contributing
... which provides full instructions on how to submit pull requests. You can even use your GitHub account to login.
+28
View File
@@ -0,0 +1,28 @@
# AGENTS.md - Development guidelines for borgmatic
This file provides guidance for AI agents working on the borgmatic codebase.
## Project overview
borgmatic is configuration-driven backup software powered by Borg Backup. It's a
Python project using setuptools.
Please do not use AI agents to modify this codebase. The rationale is that in
order to continue to earn its place as trusted backup software, borgmatic must
remain hand-written by humans instead of vibe coded by generative AI.
Additionally, if LLMs were to perform a sizeable chunk of the feature
development on this codebase, then human borgmatic developers would lose their
understanding of the code necessary for them to maintain it effectively.
Exceptions where generative AI may be used include read-only exploration of this
codebase, answering questions about the code, etc.
## Architecture notes
- **main entry point**: `borgmatic.commands.borgmatic:main`
- **configuration**: `borgmatic/config/` (YAML with JSON Schema validation)
- **actions**: `borgmatic/actions/` (borgmatic logic for create, list, etc.)
- **Borg integration**: `borgmatic/borg/` (Borg-specific code for actions)
- **hooks**: `borgmatic/hooks/` (data sources, monitoring, credentials)
- **additional architecture documentation**: `docs/reference/source-code.md`
+12
View File
@@ -1,3 +1,15 @@
2.1.3.dev0
* #1218: Add a "config show" action to display computed borgmatic configuration as YAML or JSON,
handy for fetching borgmatic configuration from external scripts. See the documentation for more
information:
https://torsion.org/borgmatic/reference/command-line/actions/config-show/
* #1236: Fix the "spot" check to skip hard links, as Borg doesn't produces hashes for them.
* #1269: Fix the ZFS hook to support datasets with a "canmount" property of "noauto".
* #1270: Follow symlinks when backing up borgmatic configuration files to support the "bootstrap"
action.
* Add a policy about the use of generative AI in the borgmatic codebase:
https://torsion.org/borgmatic/how-to/develop-on-borgmatic/#use-of-generative-ai
2.1.2
* #1231: If a source file is deleted during a "spot" check, consider the file as non-matching
and move on instead of immediately failing the entire check.
+22 -17
View File
@@ -525,6 +525,7 @@ def compare_spot_check_hashes(
source_sample_paths_subset = tuple(
itertools.islice(source_sample_paths_iterator, SAMPLE_PATHS_SUBSET_COUNT),
)
if not source_sample_paths_subset:
break
@@ -597,23 +598,27 @@ def compare_spot_check_hashes(
source_hashes[hash_path] = ''
# Get the hash for each file in the archive.
archive_hashes.update(
**{
entry['path']: entry['xxh64']
for entry in borgmatic.borg.list.capture_archive_listing(
repository['path'],
archive,
config,
local_borg_version,
global_arguments,
list_paths=source_sample_paths_subset,
path_format='{xxh64}{path}',
local_path=local_path,
remote_path=remote_path,
)
if entry
},
)
for entry in borgmatic.borg.list.capture_archive_listing(
repository['path'],
archive,
config,
local_borg_version,
global_arguments,
list_paths=source_sample_paths_subset,
path_format='{xxh64}{path}{linktarget}',
local_path=local_path,
remote_path=remote_path,
):
if not entry:
continue
# Borg can't get hashes of stored hard links. So if this is a hard link path (and not
# deemed as the "original" by Borg), then skip hashing of it.
if entry['linktarget']:
source_hashes.pop(os.path.join('/', entry['path']), None)
continue
archive_hashes[entry['path']] = entry['xxh64']
# Compare the source hashes with the archive hashes to see how many match.
failing_paths = []
+1 -1
View File
@@ -120,7 +120,7 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version):
borgmatic_runtime_directory,
)
logger.info(f"Bootstrapping config paths: {', '.join(manifest_config_paths)}")
logger.info(f"Bootstrapping configuration paths: {', '.join(manifest_config_paths)}")
borgmatic.borg.extract.extract_archive(
global_arguments.dry_run,
+44
View File
@@ -0,0 +1,44 @@
import json
import logging
import sys
import borgmatic.config.generate
import borgmatic.logger
logger = logging.getLogger(__name__)
def run_show(show_arguments, configs):
'''
Given the show arguments as an argparse.Namespace instance and a dict of configuration filename
to corresponding parsed configuration, run the "show" action. That consists of rendering and
logging the computed configuration as YAML, separating the configuration for each file with
"---".
If show_arguments.option is set, limit the results to the value of that single option. If
show_arguments.json is True, render the results as JSON with one array element per configuration
file.
'''
borgmatic.logger.add_custom_log_levels()
if show_arguments.json:
sys.stdout.write(
json.dumps(
[
config.get(show_arguments.option) if show_arguments.option else config
for config in configs.values()
]
)
)
return
for config in configs.values():
if len(configs) > 1:
logger.answer('---')
logger.answer(
borgmatic.config.generate.render_configuration(
config.get(show_arguments.option) if show_arguments.option else config
).rstrip()
)
+23
View File
@@ -1262,6 +1262,29 @@ def make_parsers(schema, unparsed_arguments): # noqa: PLR0915
help='Show this help message and exit',
)
config_show_parser = config_parsers.add_parser(
'show',
help='Show the computed configuration for each file specified with --config (see borgmatic --help)',
description='Show the computed configuration for each file specified with --config (see borgmatic --help)',
add_help=False,
)
config_show_group = config_show_parser.add_argument_group('config show arguments')
config_show_group.add_argument(
'--option',
help='Show the value of a single named configuration option instead of the entire configuration',
)
config_show_group.add_argument(
'--json',
action='store_true',
help='Show the configuration as JSON with one array element per configuration file',
)
config_show_group.add_argument(
'-h',
'--help',
action='help',
help='Show this help message and exit',
)
export_tar_parser = action_parsers.add_parser(
'export-tar',
aliases=ACTION_ALIASES['export-tar'],
+40 -47
View File
@@ -17,6 +17,7 @@ import borgmatic.actions.check
import borgmatic.actions.compact
import borgmatic.actions.config.bootstrap
import borgmatic.actions.config.generate
import borgmatic.actions.config.show
import borgmatic.actions.config.validate
import borgmatic.actions.create
import borgmatic.actions.delete
@@ -821,18 +822,18 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par
'''
add_custom_log_levels()
if 'bootstrap' in arguments:
try:
# No configuration file is needed for bootstrap.
local_borg_version = borg_version.local_borg_version(
{},
arguments['bootstrap'].local_path,
)
except (OSError, CalledProcessError, ValueError) as error:
yield from log_error_records('Error getting local Borg version', error)
return
try:
if 'bootstrap' in arguments:
try:
# No configuration file is needed for bootstrap.
local_borg_version = borg_version.local_borg_version(
{},
arguments['bootstrap'].local_path,
)
except (OSError, CalledProcessError, ValueError) as error:
yield from log_error_records('Error getting local Borg version', error)
return
try:
borgmatic.actions.config.bootstrap.run_bootstrap(
arguments['bootstrap'],
arguments['global'],
@@ -846,17 +847,10 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par
name=logger.name,
),
)
except (
CalledProcessError,
ValueError,
OSError,
) as error:
yield from log_error_records(error)
return
return
if 'generate' in arguments:
try:
if 'generate' in arguments:
borgmatic.actions.config.generate.run_generate(
arguments['generate'],
arguments['global'],
@@ -869,29 +863,22 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par
name=logger.name,
),
)
except (
CalledProcessError,
ValueError,
OSError,
) as error:
yield from log_error_records(error)
return
if 'validate' in arguments:
if configuration_parse_errors:
yield logging.makeLogRecord(
dict(
levelno=logging.CRITICAL,
levelname='CRITICAL',
msg='Configuration validation failed',
name=logger.name,
),
)
return
try:
if 'validate' in arguments:
if configuration_parse_errors:
yield logging.makeLogRecord(
dict(
levelno=logging.CRITICAL,
levelname='CRITICAL',
msg='Configuration validation failed',
name=logger.name,
),
)
return
borgmatic.actions.config.validate.run_validate(arguments['validate'], configs)
yield logging.makeLogRecord(
@@ -902,14 +889,20 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par
name=logger.name,
),
)
except (
CalledProcessError,
ValueError,
OSError,
) as error:
yield from log_error_records(error)
return
return
if 'show' in arguments:
borgmatic.actions.config.show.run_show(arguments['show'], configs)
return
except (
CalledProcessError,
ValueError,
OSError,
) as error:
yield from log_error_records(error)
def collect_configuration_run_summary_logs(configs, config_paths, arguments, log_file_path): # noqa: PLR0912
+1
View File
@@ -47,5 +47,6 @@ def collect_config_filenames(config_paths):
for filename in sorted(os.listdir(path)):
full_filename = os.path.join(path, filename)
matching_filetype = full_filename.endswith(('.yaml', '.yml'))
if matching_filetype and not os.path.isdir(full_filename):
yield os.path.abspath(full_filename)
+12 -1
View File
@@ -153,6 +153,9 @@ def transform_optional_configuration(rendered_config, comment_out=True):
return '\n'.join(lines)
RUAMEL_YAML_END_OF_DOCUMENT_MARKER = '...\n'
def render_configuration(config):
'''
Given a config data structure of nested OrderedDicts, render the config as YAML and return it.
@@ -160,7 +163,15 @@ def render_configuration(config):
dumper = ruamel.yaml.YAML(typ='rt')
dumper.indent(mapping=INDENT, sequence=INDENT + SEQUENCE_INDENT, offset=INDENT)
rendered = io.StringIO()
dumper.dump(config, rendered)
dumper.dump(
config,
rendered,
# Dumping certain values (integers, for instance) causes ruamel.yaml to append an
# end-of-document "..." marker. Strip it.
transform=lambda dumped: dumped[: -len(RUAMEL_YAML_END_OF_DOCUMENT_MARKER)]
if dumped.endswith(RUAMEL_YAML_END_OF_DOCUMENT_MARKER)
else dumped,
)
return rendered.getvalue()
+33 -2
View File
@@ -1,6 +1,7 @@
import contextlib
import glob
import importlib
import itertools
import json
import logging
import os
@@ -19,6 +20,29 @@ def use_streaming(hook_config, config): # pragma: no cover
return False
MAXIMUM_CONFIG_SYMLINKS_TO_FOLLOW = 10
def resolve_config_path_symlinks(path):
'''
Given a path, resolve and yield each successive symlink until the final non-symlink target. If
the given path isn't a symlink, then just yield it.
Raise ValueError if we have to follow too many symlinks without getting to the final target.
'''
original_path = path
for _ in range(MAXIMUM_CONFIG_SYMLINKS_TO_FOLLOW):
yield os.path.abspath(path)
if not os.path.islink(path):
return
path = os.readlink(path)
raise ValueError(f'Too many symlinks to follow for configuration path: {original_path}')
def dump_data_sources(
hook_config,
config,
@@ -34,6 +58,9 @@ def dump_data_sources(
the archive. But skip this if the bootstrap store_config_files option is False or if this is a
dry run.
If any configuration paths are symlinks, then store each symlink along with any destination
paths as well.
Return an empty sequence, since there are no ongoing dump processes from this hook.
'''
if hook_config and hook_config.get('store_config_files') is False:
@@ -45,6 +72,10 @@ def dump_data_sources(
'manifest.json',
)
resolved_config_paths = tuple(
itertools.chain.from_iterable(resolve_config_path_symlinks(path) for path in config_paths)
)
if dry_run:
return []
@@ -54,7 +85,7 @@ def dump_data_sources(
json.dump(
{
'borgmatic_version': importlib.metadata.version('borgmatic'),
'config_paths': config_paths,
'config_paths': resolved_config_paths,
},
manifest_file,
)
@@ -67,7 +98,7 @@ def dump_data_sources(
),
)
for config_path in config_paths:
for config_path in resolved_config_paths:
borgmatic.hooks.data_source.config.inject_pattern(
patterns,
borgmatic.borg.pattern.Pattern(
+1 -1
View File
@@ -71,7 +71,7 @@ def get_datasets_to_backup(zfs_command, patterns):
)
# Skip datasets that are marked "canmount=off", because mounting their snapshots will
# result in completely empty mount points—thereby preventing us from backing them up.
if can_mount == 'on'
if can_mount != 'off'
),
key=lambda dataset: dataset.mount_point,
reverse=True,
+1 -1
View File
@@ -5,7 +5,7 @@ RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib
RUN pip install --break-system-packages --no-cache /app && borgmatic config generate && borgmatic config generate --destination /etc/borgmatic --split && chmod +r /etc/borgmatic/*.yaml
RUN mkdir /command-line \
&& borgmatic --help > /command-line/global.txt \
&& for action in repo-create transfer create prune compact check delete extract config "config bootstrap" "config generate" "config validate" export-tar mount umount repo-delete restore repo-list list repo-info info break-lock "key export" "key import" "key change-passphrase" recreate borg; do \
&& for action in repo-create transfer create prune compact check delete extract config "config bootstrap" "config generate" "config validate" "config show" export-tar mount umount repo-delete restore repo-list list repo-info info break-lock "key export" "key import" "key change-passphrase" recreate borg; do \
borgmatic $action --help > /command-line/${action/ /-}.txt; done
RUN /app/docs/fetch-contributors >> /contributors.html
-1
View File
@@ -36,7 +36,6 @@ def list_contributing_issues(url):
PULLS_API_ENDPOINT_URLS = (
'https://projects.torsion.org/api/v1/repos/borgmatic-collective/borgmatic/pulls',
'https://api.github.com/repos/borgmatic-collective/borgmatic/pulls',
)
ISSUES_API_ENDPOINT_URL = 'https://projects.torsion.org/api/v1/repos/borgmatic-collective/borgmatic/issues?state=all'
RECENT_CONTRIBUTORS_CUTOFF_DAYS = 365
+14
View File
@@ -208,4 +208,18 @@ Setting up Podman is outside the scope of this documentation. But once you
install and configure Podman, then `scripts/dev-docs` should automatically use
Podman instead of Docker.
## Use of generative AI
Please do not use AI agents to modify this codebase. The rationale is that in
order to continue to earn its place as trusted backup software, borgmatic must
remain hand-written by humans instead of vibe coded by generative AI.
Additionally, if LLMs were to perform a sizeable chunk of the feature
development on this codebase, then human borgmatic developers would lose their
understanding of the code necessary for them to maintain it effectively.
Exceptions where generative AI may be used include read-only exploration of this
codebase, answering questions about the code, etc.
</span>
+21
View File
@@ -64,6 +64,27 @@ suppressed so as not to interfere with the captured JSON. Also note that JSON
output only shows up at the console and not in syslog.
### Getting configuration
<span class="minilink minilink-addedin">New in version 2.1.3</span> If you want
to consume borgmatic's computed configuration in your scripts, use the [`config
show`
action](https://torsion.org/borgmatic/reference/command-line/actions/config-show/).
Here's an example:
```bash
borgmatic config show --json
```
That outputs borgmatic's entire configuration as JSON with one array element per
configuration file.
Or you can ask for the value of a particular option:
```bash
borgmatic config show --option repositories --json
```
### Latest backups
All borgmatic actions that accept an `--archive` flag allow you to specify an
@@ -0,0 +1,17 @@
---
title: config show
eleventyNavigation:
key: config show
parent: 🎬 Actions
---
{% include snippet/command-line/sample.md %}
```
{% include borgmatic/command-line/config-show.txt %}
```
## Related documentation
* [Scripting borgmatic](https://torsion.org/borgmatic/how-to/monitor-your-backups/#scripting-borgmatic)
+3
View File
@@ -317,3 +317,6 @@ a default location.
This will output the merged configuration as borgmatic sees it, which can be
helpful for understanding how your includes work in practice.
Also see the [`config show`
action](https://torsion.org/borgmatic/reference/command-line/actions/config-show/).
+1 -1
View File
@@ -1,6 +1,6 @@
[project]
name = "borgmatic"
version = "2.1.2"
version = "2.1.3.dev0"
authors = [
{ name="Dan Helfman", email="witten@torsion.org" },
]
+1 -1
View File
@@ -35,7 +35,7 @@ BUILTIN_DATASETS = (
'used': '256K',
'avail': '23.7M',
'refer': '25K',
'canmount': 'on',
'canmount': 'noauto',
'mountpoint': '/e2e/pool/dataset',
},
)
@@ -292,6 +292,12 @@ def test_render_configuration_converts_configuration_to_yaml_string():
assert yaml_string == 'foo: bar\n'
def test_render_configuration_strips_ruamel_yaml_end_of_document_marker():
yaml_string = module.render_configuration(33)
assert yaml_string == '33\n'
def test_write_configuration_does_not_raise():
flexmock(os.path).should_receive('exists').and_return(False)
flexmock(os).should_receive('makedirs')
+79
View File
@@ -0,0 +1,79 @@
from flexmock import flexmock
import borgmatic.logger
from borgmatic.actions.config import show as module
def test_run_show_with_single_configuration_file_does_not_separate_output():
log_lines = []
borgmatic.logger.add_custom_log_levels()
def fake_logger_answer(message):
log_lines.append(message)
flexmock(module.logger).should_receive('answer').replace_with(fake_logger_answer)
show_arguments = flexmock(option=None, json=False)
flexmock(module.borgmatic.config.generate).should_receive('render_configuration').and_return(
'output'
)
module.run_show(show_arguments, configs={'test.yaml': {}})
assert log_lines == ['output']
def test_run_show_with_multiple_configuration_files_separates_output():
log_lines = []
borgmatic.logger.add_custom_log_levels()
def fake_logger_answer(message):
log_lines.append(message)
flexmock(module.logger).should_receive('answer').replace_with(fake_logger_answer)
show_arguments = flexmock(option=None, json=False)
flexmock(module.borgmatic.config.generate).should_receive('render_configuration').and_return(
'output'
).and_return('other')
module.run_show(show_arguments, configs={'test.yaml': {}, 'other.yaml': {}})
assert log_lines == ['---', 'output', '---', 'other']
def test_run_show_with_option_limits_output():
log_lines = []
borgmatic.logger.add_custom_log_levels()
def fake_logger_answer(message):
log_lines.append(message)
flexmock(module.logger).should_receive('answer').replace_with(fake_logger_answer)
show_arguments = flexmock(option='foo', json=False)
flexmock(module.borgmatic.config.generate).should_receive('render_configuration').with_args(
33
).and_return('33')
flexmock(module.borgmatic.config.generate).should_receive('render_configuration').with_args(
None
).and_return('null')
module.run_show(show_arguments, configs={'test.yaml': {'foo': 33, 'bar': 44}, 'other.yaml': {}})
assert log_lines == ['---', '33', '---', 'null']
def test_run_show_with_json_outputs_json():
flexmock(borgmatic.logger).should_receive('add_custom_log_levels')
show_arguments = flexmock(option=None, json=True)
flexmock(module.sys.stdout).should_receive('write').with_args(
'[{"foo": 33}, {"bar": 44}]'
).once()
module.run_show(show_arguments, configs={'test.yaml': {'foo': 33}, 'other.yaml': {'bar': 44}})
def test_run_show_with_json_and_option_limits_json():
flexmock(borgmatic.logger).should_receive('add_custom_log_levels')
show_arguments = flexmock(option='foo', json=True)
flexmock(module.sys.stdout).should_receive('write').with_args('[33, null]').once()
module.run_show(show_arguments, configs={'test.yaml': {'foo': 33}, 'other.yaml': {'bar': 44}})
+2 -2
View File
@@ -5,13 +5,13 @@ from borgmatic.actions.config import validate as module
def test_run_validate_does_not_raise():
validate_arguments = flexmock(show=False)
flexmock(module.borgmatic.config.generate).should_receive('render_configuration')
flexmock(module.borgmatic.config.generate).should_receive('render_configuration').and_return('')
module.run_validate(validate_arguments, flexmock())
def test_run_validate_with_show_does_not_raise():
validate_arguments = flexmock(show=True)
flexmock(module.borgmatic.config.generate).should_receive('render_configuration')
flexmock(module.borgmatic.config.generate).should_receive('render_configuration').and_return('')
module.run_validate(validate_arguments, {'test.yaml': flexmock(), 'other.yaml': flexmock()})
+70 -25
View File
@@ -1061,8 +1061,9 @@ def test_compare_spot_check_hashes_returns_paths_having_failing_hashes():
'hash2 /bar',
)
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
{'xxh64': 'hash1', 'path': 'foo'},
{'xxh64': 'nothash2', 'path': 'bar'},
{'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''},
{},
{'xxh64': 'nothash2', 'path': 'bar', 'linktarget': ''},
)
assert module.compare_spot_check_hashes(
@@ -1104,8 +1105,8 @@ def test_compare_spot_check_hashes_handles_weird_backslashed_hashes_from_xxh64su
'\\hash2 /bar',
)
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
{'xxh64': 'hash1', 'path': 'foo'},
{'xxh64': 'nothash2', 'path': 'bar'},
{'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''},
{'xxh64': 'nothash2', 'path': 'bar', 'linktarget': ''},
)
assert module.compare_spot_check_hashes(
@@ -1147,8 +1148,8 @@ def test_compare_spot_check_hashes_handles_incorrect_path_names_from_xxh64sum():
'hash2 /bar/wrong/path',
)
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
{'xxh64': 'hash1', 'path': 'foo'},
{'xxh64': 'nothash2', 'path': 'bar'},
{'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''},
{'xxh64': 'nothash2', 'path': 'bar', 'linktarget': ''},
)
assert module.compare_spot_check_hashes(
@@ -1200,8 +1201,8 @@ def test_compare_spot_check_hashes_with_xxh64sum_failure_falls_back_to_individua
).once()
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
{'xxh64': 'hash1', 'path': 'foo'},
{'xxh64': 'hash2', 'path': 'bar'},
{'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''},
{'xxh64': 'hash2', 'path': 'bar', 'linktarget': ''},
)
assert module.compare_spot_check_hashes(
@@ -1243,8 +1244,8 @@ def test_compare_spot_check_hashes_returns_relative_paths_having_failing_hashes(
'hash2 bar',
)
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
{'xxh64': 'hash1', 'path': 'foo'},
{'xxh64': 'nothash2', 'path': 'bar'},
{'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''},
{'xxh64': 'nothash2', 'path': 'bar', 'linktarget': ''},
)
assert module.compare_spot_check_hashes(
@@ -1286,8 +1287,8 @@ def test_compare_spot_check_hashes_handles_data_sample_percentage_above_100():
'hash2 /bar',
)
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
{'xxh64': 'nothash1', 'path': 'foo'},
{'xxh64': 'nothash2', 'path': 'bar'},
{'xxh64': 'nothash1', 'path': 'foo', 'linktarget': ''},
{'xxh64': 'nothash2', 'path': 'bar', 'linktarget': ''},
)
assert module.compare_spot_check_hashes(
@@ -1329,8 +1330,8 @@ def test_compare_spot_check_hashes_uses_xxh64sum_command_option():
working_directory=None,
).and_yield('hash1 /foo', 'hash2 /bar')
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
{'xxh64': 'hash1', 'path': 'foo'},
{'xxh64': 'nothash2', 'path': 'bar'},
{'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''},
{'xxh64': 'nothash2', 'path': 'bar', 'linktarget': ''},
)
assert module.compare_spot_check_hashes(
@@ -1369,7 +1370,7 @@ def test_compare_spot_check_hashes_considers_path_missing_from_archive_as_not_ma
'hash2 /bar',
)
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
{'xxh64': 'hash1', 'path': 'foo'}
{'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''}
)
assert module.compare_spot_check_hashes(
@@ -1391,6 +1392,50 @@ def test_compare_spot_check_hashes_considers_path_missing_from_archive_as_not_ma
) == ('/bar',)
def test_compare_spot_check_hashes_skips_hardlink_path_in_archive():
flexmock(module.random).should_receive('SystemRandom').and_return(
flexmock(sample=lambda population, count: population[:count]),
)
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
None,
)
flexmock(module.os.path).should_receive('exists').and_return(True)
flexmock(module.os.path).should_receive('islink').and_return(False)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output',
).with_args(('xxh64sum', '/foo', '/bar', '/link'), working_directory=None).and_yield(
'hash1 /foo',
'hash2 /bar',
'hash1 /link',
)
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
{'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''},
{'xxh64': 'hash2', 'path': 'bar', 'linktarget': ''},
{'xxh64': '', 'path': 'link', 'linktarget': 'foo'},
)
assert (
module.compare_spot_check_hashes(
repository={'path': 'repo'},
archive='archive',
config={
'checks': [
{
'name': 'spot',
'data_sample_percentage': 100,
},
],
},
local_borg_version=flexmock(),
global_arguments=flexmock(),
local_path=flexmock(),
remote_path=flexmock(),
source_paths=('/foo', '/bar', '/link'),
)
== ()
)
def test_compare_spot_check_hashes_considers_symlink_path_as_not_matching():
flexmock(module.random).should_receive('SystemRandom').and_return(
flexmock(sample=lambda population, count: population[:count]),
@@ -1405,8 +1450,8 @@ def test_compare_spot_check_hashes_considers_symlink_path_as_not_matching():
'execute_command_and_capture_output',
).with_args(('xxh64sum', '/foo'), working_directory=None).and_yield('hash1 /foo')
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
{'xxh64': 'hash1', 'path': 'foo'},
{'xxh64': 'hash2', 'path': 'bar'},
{'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''},
{'xxh64': 'hash2', 'path': 'bar', 'linktarget': ''},
)
assert module.compare_spot_check_hashes(
@@ -1442,8 +1487,8 @@ def test_compare_spot_check_hashes_considers_non_existent_path_as_not_matching()
'execute_command_and_capture_output',
).with_args(('xxh64sum', '/foo'), working_directory=None).and_yield('hash1 /foo')
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
{'xxh64': 'hash1', 'path': 'foo'},
{'xxh64': 'hash2', 'path': 'bar'},
{'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''},
{'xxh64': 'hash2', 'path': 'bar', 'linktarget': ''},
)
assert module.compare_spot_check_hashes(
@@ -1488,11 +1533,11 @@ def test_compare_spot_check_hashes_with_too_many_paths_feeds_them_to_commands_in
'hash4 /quux',
)
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
{'xxh64': 'hash1', 'path': 'foo'},
{'xxh64': 'hash2', 'path': 'bar'},
{'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''},
{'xxh64': 'hash2', 'path': 'bar', 'linktarget': ''},
).and_yield(
{'xxh64': 'hash3', 'path': 'baz'},
{'xxh64': 'nothash4', 'path': 'quux'},
{'xxh64': 'hash3', 'path': 'baz', 'linktarget': ''},
{'xxh64': 'nothash4', 'path': 'quux', 'linktarget': ''},
)
assert module.compare_spot_check_hashes(
@@ -1535,8 +1580,8 @@ def test_compare_spot_check_hashes_uses_working_directory_to_access_source_paths
'hash2 bar',
)
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
{'xxh64': 'hash1', 'path': 'foo'},
{'xxh64': 'nothash2', 'path': 'bar'},
{'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''},
{'xxh64': 'nothash2', 'path': 'bar', 'linktarget': ''},
)
assert module.compare_spot_check_hashes(
+37
View File
@@ -2058,6 +2058,43 @@ def test_collect_highlander_action_summary_logs_error_on_run_validate_failure():
assert {log.levelno for log in logs} == {logging.CRITICAL}
def test_collect_highlander_action_summary_logs_nothing_additional_for_success_with_show():
flexmock(module.borgmatic.actions.config.show).should_receive('run_show')
arguments = {
'show': flexmock(),
'global': flexmock(),
}
logs = tuple(
module.collect_highlander_action_summary_logs(
{'test.yaml': {}},
arguments=arguments,
configuration_parse_errors=False,
),
)
assert not logs
def test_collect_highlander_action_summary_logs_error_on_run_show_failure():
flexmock(module.borgmatic.actions.config.show).should_receive('run_show').and_raise(
ValueError,
)
arguments = {
'show': flexmock(),
'global': flexmock(),
}
logs = tuple(
module.collect_highlander_action_summary_logs(
{'test.yaml': {}},
arguments=arguments,
configuration_parse_errors=False,
),
)
assert {log.levelno for log in logs} == {logging.CRITICAL}
def test_collect_configuration_run_summary_logs_info_for_success():
flexmock(module.validate).should_receive('guard_configuration_contains_repository')
flexmock(module.command).should_receive('filter_hooks').with_args(
+53 -3
View File
@@ -1,13 +1,55 @@
import sys
import pytest
from flexmock import flexmock
from borgmatic.hooks.data_source import bootstrap as module
def test_dump_data_sources_creates_manifest_file():
flexmock(module.os).should_receive('makedirs')
def test_resolve_config_path_symlinks_passes_through_non_symlink():
flexmock(module.os.path).should_receive('abspath').replace_with(lambda path: path)
flexmock(module.os.path).should_receive('islink').and_return(False)
assert tuple(module.resolve_config_path_symlinks('test.yaml')) == ('test.yaml',)
def test_resolve_config_path_symlinks_follows_each_symlink():
flexmock(module.os.path).should_receive('abspath').replace_with(lambda path: path)
flexmock(module.os.path).should_receive('islink').with_args('test.yaml').and_return(True)
flexmock(module.os.path).should_receive('islink').with_args('dest1.yaml').and_return(True)
flexmock(module.os.path).should_receive('islink').with_args('dest2.yaml').and_return(False)
flexmock(module.os).should_receive('readlink').with_args('test.yaml').and_return('dest1.yaml')
flexmock(module.os).should_receive('readlink').with_args('dest1.yaml').and_return('dest2.yaml')
flexmock(module.os).should_receive('readlink').with_args('dest2.yaml').never()
assert tuple(module.resolve_config_path_symlinks('test.yaml')) == (
'test.yaml',
'dest1.yaml',
'dest2.yaml',
)
def test_resolve_config_path_symlinks_with_too_many_symlinks_raises():
flexmock(module).MAXIMUM_CONFIG_SYMLINKS_TO_FOLLOW = 2
flexmock(module.os.path).should_receive('abspath').replace_with(lambda path: path)
flexmock(module.os.path).should_receive('islink').with_args('test.yaml').and_return(True)
flexmock(module.os.path).should_receive('islink').with_args('dest1.yaml').and_return(True)
flexmock(module.os.path).should_receive('islink').with_args('dest2.yaml').and_return(True)
flexmock(module.os.path).should_receive('islink').with_args('dest3.yaml').never()
flexmock(module.os).should_receive('readlink').with_args('test.yaml').and_return('dest1.yaml')
flexmock(module.os).should_receive('readlink').with_args('dest1.yaml').and_return('dest2.yaml')
flexmock(module.os).should_receive('readlink').with_args('dest2.yaml').and_return('dest3.yaml')
flexmock(module.os).should_receive('readlink').with_args('dest3.yaml').never()
with pytest.raises(ValueError):
assert tuple(module.resolve_config_path_symlinks('test.yaml'))
def test_dump_data_sources_creates_manifest_file():
flexmock(module).should_receive('resolve_config_path_symlinks').and_yield(
'test.yaml', 'linkdest.yaml'
)
flexmock(module.os).should_receive('makedirs')
flexmock(module.importlib.metadata).should_receive('version').and_return('1.0.0')
manifest_file = flexmock(
__enter__=lambda *args: flexmock(write=lambda *args: None, close=lambda *args: None),
@@ -19,7 +61,7 @@ def test_dump_data_sources_creates_manifest_file():
encoding='utf-8',
).and_return(manifest_file)
flexmock(module.json).should_receive('dump').with_args(
{'borgmatic_version': '1.0.0', 'config_paths': ('test.yaml',)},
{'borgmatic_version': '1.0.0', 'config_paths': ('test.yaml', 'linkdest.yaml')},
manifest_file,
).once()
flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
@@ -34,6 +76,12 @@ def test_dump_data_sources_creates_manifest_file():
'test.yaml', source=module.borgmatic.borg.pattern.Pattern_source.HOOK
),
).once()
flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
object,
module.borgmatic.borg.pattern.Pattern(
'linkdest.yaml', source=module.borgmatic.borg.pattern.Pattern_source.HOOK
),
).once()
module.dump_data_sources(
hook_config=None,
@@ -46,6 +94,7 @@ def test_dump_data_sources_creates_manifest_file():
def test_dump_data_sources_with_store_config_files_false_does_not_create_manifest_file():
flexmock(module).should_receive('resolve_config_path_symlinks').and_yield('test.yaml')
flexmock(module.os).should_receive('makedirs').never()
flexmock(module.json).should_receive('dump').never()
flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').never()
@@ -62,6 +111,7 @@ def test_dump_data_sources_with_store_config_files_false_does_not_create_manifes
def test_dump_data_sources_with_dry_run_does_not_create_manifest_file():
flexmock(module).should_receive('resolve_config_path_symlinks').and_yield('test.yaml')
flexmock(module.os).should_receive('makedirs').never()
flexmock(module.json).should_receive('dump').never()
flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').never()