mirror of
https://github.com/borgmatic-collective/borgmatic.git
synced 2026-08-06 16:23:00 +02:00
Merge branch 'main' into add-diff
This commit is contained in:
@@ -1,10 +0,0 @@
|
||||
## Hold up, GitHub users
|
||||
|
||||
Thanks for your contribution!
|
||||
|
||||
Unfortunately, we don't use GitHub pull requests to manage code contributions to this repository (and GitHub doesn't have any way to disable pull requests entirely). Instead, please see:
|
||||
|
||||
https://torsion.org/borgmatic/#contributing
|
||||
|
||||
... which provides full instructions on how to submit pull requests. You can even use your GitHub account to login.
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
# AGENTS.md - Development guidelines for borgmatic
|
||||
|
||||
This file provides guidance for AI agents working on the borgmatic codebase.
|
||||
|
||||
## Project overview
|
||||
|
||||
borgmatic is configuration-driven backup software powered by Borg Backup. It's a
|
||||
Python project using setuptools.
|
||||
|
||||
Please do not use AI agents to modify this codebase. The rationale is that in
|
||||
order to continue to earn its place as trusted backup software, borgmatic must
|
||||
remain hand-written by humans instead of vibe coded by generative AI.
|
||||
|
||||
Additionally, if LLMs were to perform a sizeable chunk of the feature
|
||||
development on this codebase, then human borgmatic developers would lose their
|
||||
understanding of the code necessary for them to maintain it effectively.
|
||||
|
||||
Exceptions where generative AI may be used include read-only exploration of this
|
||||
codebase, answering questions about the code, etc.
|
||||
|
||||
## Architecture notes
|
||||
|
||||
- **main entry point**: `borgmatic.commands.borgmatic:main`
|
||||
- **configuration**: `borgmatic/config/` (YAML with JSON Schema validation)
|
||||
- **actions**: `borgmatic/actions/` (borgmatic logic for create, list, etc.)
|
||||
- **Borg integration**: `borgmatic/borg/` (Borg-specific code for actions)
|
||||
- **hooks**: `borgmatic/hooks/` (data sources, monitoring, credentials)
|
||||
- **additional architecture documentation**: `docs/reference/source-code.md`
|
||||
@@ -1,3 +1,15 @@
|
||||
2.1.3.dev0
|
||||
* #1218: Add a "config show" action to display computed borgmatic configuration as YAML or JSON,
|
||||
handy for fetching borgmatic configuration from external scripts. See the documentation for more
|
||||
information:
|
||||
https://torsion.org/borgmatic/reference/command-line/actions/config-show/
|
||||
* #1236: Fix the "spot" check to skip hard links, as Borg doesn't produces hashes for them.
|
||||
* #1269: Fix the ZFS hook to support datasets with a "canmount" property of "noauto".
|
||||
* #1270: Follow symlinks when backing up borgmatic configuration files to support the "bootstrap"
|
||||
action.
|
||||
* Add a policy about the use of generative AI in the borgmatic codebase:
|
||||
https://torsion.org/borgmatic/how-to/develop-on-borgmatic/#use-of-generative-ai
|
||||
|
||||
2.1.2
|
||||
* #1231: If a source file is deleted during a "spot" check, consider the file as non-matching
|
||||
and move on instead of immediately failing the entire check.
|
||||
|
||||
+22
-17
@@ -525,6 +525,7 @@ def compare_spot_check_hashes(
|
||||
source_sample_paths_subset = tuple(
|
||||
itertools.islice(source_sample_paths_iterator, SAMPLE_PATHS_SUBSET_COUNT),
|
||||
)
|
||||
|
||||
if not source_sample_paths_subset:
|
||||
break
|
||||
|
||||
@@ -597,23 +598,27 @@ def compare_spot_check_hashes(
|
||||
source_hashes[hash_path] = ''
|
||||
|
||||
# Get the hash for each file in the archive.
|
||||
archive_hashes.update(
|
||||
**{
|
||||
entry['path']: entry['xxh64']
|
||||
for entry in borgmatic.borg.list.capture_archive_listing(
|
||||
repository['path'],
|
||||
archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
list_paths=source_sample_paths_subset,
|
||||
path_format='{xxh64}{path}',
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
)
|
||||
if entry
|
||||
},
|
||||
)
|
||||
for entry in borgmatic.borg.list.capture_archive_listing(
|
||||
repository['path'],
|
||||
archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
list_paths=source_sample_paths_subset,
|
||||
path_format='{xxh64}{path}{linktarget}',
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
):
|
||||
if not entry:
|
||||
continue
|
||||
|
||||
# Borg can't get hashes of stored hard links. So if this is a hard link path (and not
|
||||
# deemed as the "original" by Borg), then skip hashing of it.
|
||||
if entry['linktarget']:
|
||||
source_hashes.pop(os.path.join('/', entry['path']), None)
|
||||
continue
|
||||
|
||||
archive_hashes[entry['path']] = entry['xxh64']
|
||||
|
||||
# Compare the source hashes with the archive hashes to see how many match.
|
||||
failing_paths = []
|
||||
|
||||
@@ -120,7 +120,7 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version):
|
||||
borgmatic_runtime_directory,
|
||||
)
|
||||
|
||||
logger.info(f"Bootstrapping config paths: {', '.join(manifest_config_paths)}")
|
||||
logger.info(f"Bootstrapping configuration paths: {', '.join(manifest_config_paths)}")
|
||||
|
||||
borgmatic.borg.extract.extract_archive(
|
||||
global_arguments.dry_run,
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
|
||||
import borgmatic.config.generate
|
||||
import borgmatic.logger
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run_show(show_arguments, configs):
|
||||
'''
|
||||
Given the show arguments as an argparse.Namespace instance and a dict of configuration filename
|
||||
to corresponding parsed configuration, run the "show" action. That consists of rendering and
|
||||
logging the computed configuration as YAML, separating the configuration for each file with
|
||||
"---".
|
||||
|
||||
If show_arguments.option is set, limit the results to the value of that single option. If
|
||||
show_arguments.json is True, render the results as JSON with one array element per configuration
|
||||
file.
|
||||
'''
|
||||
borgmatic.logger.add_custom_log_levels()
|
||||
|
||||
if show_arguments.json:
|
||||
sys.stdout.write(
|
||||
json.dumps(
|
||||
[
|
||||
config.get(show_arguments.option) if show_arguments.option else config
|
||||
for config in configs.values()
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
return
|
||||
|
||||
for config in configs.values():
|
||||
if len(configs) > 1:
|
||||
logger.answer('---')
|
||||
|
||||
logger.answer(
|
||||
borgmatic.config.generate.render_configuration(
|
||||
config.get(show_arguments.option) if show_arguments.option else config
|
||||
).rstrip()
|
||||
)
|
||||
@@ -1262,6 +1262,29 @@ def make_parsers(schema, unparsed_arguments): # noqa: PLR0915
|
||||
help='Show this help message and exit',
|
||||
)
|
||||
|
||||
config_show_parser = config_parsers.add_parser(
|
||||
'show',
|
||||
help='Show the computed configuration for each file specified with --config (see borgmatic --help)',
|
||||
description='Show the computed configuration for each file specified with --config (see borgmatic --help)',
|
||||
add_help=False,
|
||||
)
|
||||
config_show_group = config_show_parser.add_argument_group('config show arguments')
|
||||
config_show_group.add_argument(
|
||||
'--option',
|
||||
help='Show the value of a single named configuration option instead of the entire configuration',
|
||||
)
|
||||
config_show_group.add_argument(
|
||||
'--json',
|
||||
action='store_true',
|
||||
help='Show the configuration as JSON with one array element per configuration file',
|
||||
)
|
||||
config_show_group.add_argument(
|
||||
'-h',
|
||||
'--help',
|
||||
action='help',
|
||||
help='Show this help message and exit',
|
||||
)
|
||||
|
||||
export_tar_parser = action_parsers.add_parser(
|
||||
'export-tar',
|
||||
aliases=ACTION_ALIASES['export-tar'],
|
||||
|
||||
@@ -17,6 +17,7 @@ import borgmatic.actions.check
|
||||
import borgmatic.actions.compact
|
||||
import borgmatic.actions.config.bootstrap
|
||||
import borgmatic.actions.config.generate
|
||||
import borgmatic.actions.config.show
|
||||
import borgmatic.actions.config.validate
|
||||
import borgmatic.actions.create
|
||||
import borgmatic.actions.delete
|
||||
@@ -821,18 +822,18 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par
|
||||
'''
|
||||
add_custom_log_levels()
|
||||
|
||||
if 'bootstrap' in arguments:
|
||||
try:
|
||||
# No configuration file is needed for bootstrap.
|
||||
local_borg_version = borg_version.local_borg_version(
|
||||
{},
|
||||
arguments['bootstrap'].local_path,
|
||||
)
|
||||
except (OSError, CalledProcessError, ValueError) as error:
|
||||
yield from log_error_records('Error getting local Borg version', error)
|
||||
return
|
||||
try:
|
||||
if 'bootstrap' in arguments:
|
||||
try:
|
||||
# No configuration file is needed for bootstrap.
|
||||
local_borg_version = borg_version.local_borg_version(
|
||||
{},
|
||||
arguments['bootstrap'].local_path,
|
||||
)
|
||||
except (OSError, CalledProcessError, ValueError) as error:
|
||||
yield from log_error_records('Error getting local Borg version', error)
|
||||
return
|
||||
|
||||
try:
|
||||
borgmatic.actions.config.bootstrap.run_bootstrap(
|
||||
arguments['bootstrap'],
|
||||
arguments['global'],
|
||||
@@ -846,17 +847,10 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par
|
||||
name=logger.name,
|
||||
),
|
||||
)
|
||||
except (
|
||||
CalledProcessError,
|
||||
ValueError,
|
||||
OSError,
|
||||
) as error:
|
||||
yield from log_error_records(error)
|
||||
|
||||
return
|
||||
return
|
||||
|
||||
if 'generate' in arguments:
|
||||
try:
|
||||
if 'generate' in arguments:
|
||||
borgmatic.actions.config.generate.run_generate(
|
||||
arguments['generate'],
|
||||
arguments['global'],
|
||||
@@ -869,29 +863,22 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par
|
||||
name=logger.name,
|
||||
),
|
||||
)
|
||||
except (
|
||||
CalledProcessError,
|
||||
ValueError,
|
||||
OSError,
|
||||
) as error:
|
||||
yield from log_error_records(error)
|
||||
|
||||
return
|
||||
|
||||
if 'validate' in arguments:
|
||||
if configuration_parse_errors:
|
||||
yield logging.makeLogRecord(
|
||||
dict(
|
||||
levelno=logging.CRITICAL,
|
||||
levelname='CRITICAL',
|
||||
msg='Configuration validation failed',
|
||||
name=logger.name,
|
||||
),
|
||||
)
|
||||
|
||||
return
|
||||
|
||||
try:
|
||||
if 'validate' in arguments:
|
||||
if configuration_parse_errors:
|
||||
yield logging.makeLogRecord(
|
||||
dict(
|
||||
levelno=logging.CRITICAL,
|
||||
levelname='CRITICAL',
|
||||
msg='Configuration validation failed',
|
||||
name=logger.name,
|
||||
),
|
||||
)
|
||||
|
||||
return
|
||||
|
||||
borgmatic.actions.config.validate.run_validate(arguments['validate'], configs)
|
||||
|
||||
yield logging.makeLogRecord(
|
||||
@@ -902,14 +889,20 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par
|
||||
name=logger.name,
|
||||
),
|
||||
)
|
||||
except (
|
||||
CalledProcessError,
|
||||
ValueError,
|
||||
OSError,
|
||||
) as error:
|
||||
yield from log_error_records(error)
|
||||
|
||||
return
|
||||
return
|
||||
|
||||
if 'show' in arguments:
|
||||
borgmatic.actions.config.show.run_show(arguments['show'], configs)
|
||||
|
||||
return
|
||||
|
||||
except (
|
||||
CalledProcessError,
|
||||
ValueError,
|
||||
OSError,
|
||||
) as error:
|
||||
yield from log_error_records(error)
|
||||
|
||||
|
||||
def collect_configuration_run_summary_logs(configs, config_paths, arguments, log_file_path): # noqa: PLR0912
|
||||
|
||||
@@ -47,5 +47,6 @@ def collect_config_filenames(config_paths):
|
||||
for filename in sorted(os.listdir(path)):
|
||||
full_filename = os.path.join(path, filename)
|
||||
matching_filetype = full_filename.endswith(('.yaml', '.yml'))
|
||||
|
||||
if matching_filetype and not os.path.isdir(full_filename):
|
||||
yield os.path.abspath(full_filename)
|
||||
|
||||
@@ -153,6 +153,9 @@ def transform_optional_configuration(rendered_config, comment_out=True):
|
||||
return '\n'.join(lines)
|
||||
|
||||
|
||||
RUAMEL_YAML_END_OF_DOCUMENT_MARKER = '...\n'
|
||||
|
||||
|
||||
def render_configuration(config):
|
||||
'''
|
||||
Given a config data structure of nested OrderedDicts, render the config as YAML and return it.
|
||||
@@ -160,7 +163,15 @@ def render_configuration(config):
|
||||
dumper = ruamel.yaml.YAML(typ='rt')
|
||||
dumper.indent(mapping=INDENT, sequence=INDENT + SEQUENCE_INDENT, offset=INDENT)
|
||||
rendered = io.StringIO()
|
||||
dumper.dump(config, rendered)
|
||||
dumper.dump(
|
||||
config,
|
||||
rendered,
|
||||
# Dumping certain values (integers, for instance) causes ruamel.yaml to append an
|
||||
# end-of-document "..." marker. Strip it.
|
||||
transform=lambda dumped: dumped[: -len(RUAMEL_YAML_END_OF_DOCUMENT_MARKER)]
|
||||
if dumped.endswith(RUAMEL_YAML_END_OF_DOCUMENT_MARKER)
|
||||
else dumped,
|
||||
)
|
||||
|
||||
return rendered.getvalue()
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import contextlib
|
||||
import glob
|
||||
import importlib
|
||||
import itertools
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -19,6 +20,29 @@ def use_streaming(hook_config, config): # pragma: no cover
|
||||
return False
|
||||
|
||||
|
||||
MAXIMUM_CONFIG_SYMLINKS_TO_FOLLOW = 10
|
||||
|
||||
|
||||
def resolve_config_path_symlinks(path):
|
||||
'''
|
||||
Given a path, resolve and yield each successive symlink until the final non-symlink target. If
|
||||
the given path isn't a symlink, then just yield it.
|
||||
|
||||
Raise ValueError if we have to follow too many symlinks without getting to the final target.
|
||||
'''
|
||||
original_path = path
|
||||
|
||||
for _ in range(MAXIMUM_CONFIG_SYMLINKS_TO_FOLLOW):
|
||||
yield os.path.abspath(path)
|
||||
|
||||
if not os.path.islink(path):
|
||||
return
|
||||
|
||||
path = os.readlink(path)
|
||||
|
||||
raise ValueError(f'Too many symlinks to follow for configuration path: {original_path}')
|
||||
|
||||
|
||||
def dump_data_sources(
|
||||
hook_config,
|
||||
config,
|
||||
@@ -34,6 +58,9 @@ def dump_data_sources(
|
||||
the archive. But skip this if the bootstrap store_config_files option is False or if this is a
|
||||
dry run.
|
||||
|
||||
If any configuration paths are symlinks, then store each symlink along with any destination
|
||||
paths as well.
|
||||
|
||||
Return an empty sequence, since there are no ongoing dump processes from this hook.
|
||||
'''
|
||||
if hook_config and hook_config.get('store_config_files') is False:
|
||||
@@ -45,6 +72,10 @@ def dump_data_sources(
|
||||
'manifest.json',
|
||||
)
|
||||
|
||||
resolved_config_paths = tuple(
|
||||
itertools.chain.from_iterable(resolve_config_path_symlinks(path) for path in config_paths)
|
||||
)
|
||||
|
||||
if dry_run:
|
||||
return []
|
||||
|
||||
@@ -54,7 +85,7 @@ def dump_data_sources(
|
||||
json.dump(
|
||||
{
|
||||
'borgmatic_version': importlib.metadata.version('borgmatic'),
|
||||
'config_paths': config_paths,
|
||||
'config_paths': resolved_config_paths,
|
||||
},
|
||||
manifest_file,
|
||||
)
|
||||
@@ -67,7 +98,7 @@ def dump_data_sources(
|
||||
),
|
||||
)
|
||||
|
||||
for config_path in config_paths:
|
||||
for config_path in resolved_config_paths:
|
||||
borgmatic.hooks.data_source.config.inject_pattern(
|
||||
patterns,
|
||||
borgmatic.borg.pattern.Pattern(
|
||||
|
||||
@@ -71,7 +71,7 @@ def get_datasets_to_backup(zfs_command, patterns):
|
||||
)
|
||||
# Skip datasets that are marked "canmount=off", because mounting their snapshots will
|
||||
# result in completely empty mount points—thereby preventing us from backing them up.
|
||||
if can_mount == 'on'
|
||||
if can_mount != 'off'
|
||||
),
|
||||
key=lambda dataset: dataset.mount_point,
|
||||
reverse=True,
|
||||
|
||||
+1
-1
@@ -5,7 +5,7 @@ RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib
|
||||
RUN pip install --break-system-packages --no-cache /app && borgmatic config generate && borgmatic config generate --destination /etc/borgmatic --split && chmod +r /etc/borgmatic/*.yaml
|
||||
RUN mkdir /command-line \
|
||||
&& borgmatic --help > /command-line/global.txt \
|
||||
&& for action in repo-create transfer create prune compact check delete extract config "config bootstrap" "config generate" "config validate" export-tar mount umount repo-delete restore repo-list list repo-info info break-lock "key export" "key import" "key change-passphrase" recreate borg; do \
|
||||
&& for action in repo-create transfer create prune compact check delete extract config "config bootstrap" "config generate" "config validate" "config show" export-tar mount umount repo-delete restore repo-list list repo-info info break-lock "key export" "key import" "key change-passphrase" recreate borg; do \
|
||||
borgmatic $action --help > /command-line/${action/ /-}.txt; done
|
||||
RUN /app/docs/fetch-contributors >> /contributors.html
|
||||
|
||||
|
||||
@@ -36,7 +36,6 @@ def list_contributing_issues(url):
|
||||
|
||||
PULLS_API_ENDPOINT_URLS = (
|
||||
'https://projects.torsion.org/api/v1/repos/borgmatic-collective/borgmatic/pulls',
|
||||
'https://api.github.com/repos/borgmatic-collective/borgmatic/pulls',
|
||||
)
|
||||
ISSUES_API_ENDPOINT_URL = 'https://projects.torsion.org/api/v1/repos/borgmatic-collective/borgmatic/issues?state=all'
|
||||
RECENT_CONTRIBUTORS_CUTOFF_DAYS = 365
|
||||
|
||||
@@ -208,4 +208,18 @@ Setting up Podman is outside the scope of this documentation. But once you
|
||||
install and configure Podman, then `scripts/dev-docs` should automatically use
|
||||
Podman instead of Docker.
|
||||
|
||||
|
||||
## Use of generative AI
|
||||
|
||||
Please do not use AI agents to modify this codebase. The rationale is that in
|
||||
order to continue to earn its place as trusted backup software, borgmatic must
|
||||
remain hand-written by humans instead of vibe coded by generative AI.
|
||||
|
||||
Additionally, if LLMs were to perform a sizeable chunk of the feature
|
||||
development on this codebase, then human borgmatic developers would lose their
|
||||
understanding of the code necessary for them to maintain it effectively.
|
||||
|
||||
Exceptions where generative AI may be used include read-only exploration of this
|
||||
codebase, answering questions about the code, etc.
|
||||
|
||||
</span>
|
||||
|
||||
@@ -64,6 +64,27 @@ suppressed so as not to interfere with the captured JSON. Also note that JSON
|
||||
output only shows up at the console and not in syslog.
|
||||
|
||||
|
||||
### Getting configuration
|
||||
|
||||
<span class="minilink minilink-addedin">New in version 2.1.3</span> If you want
|
||||
to consume borgmatic's computed configuration in your scripts, use the [`config
|
||||
show`
|
||||
action](https://torsion.org/borgmatic/reference/command-line/actions/config-show/).
|
||||
Here's an example:
|
||||
|
||||
```bash
|
||||
borgmatic config show --json
|
||||
```
|
||||
|
||||
That outputs borgmatic's entire configuration as JSON with one array element per
|
||||
configuration file.
|
||||
|
||||
Or you can ask for the value of a particular option:
|
||||
|
||||
```bash
|
||||
borgmatic config show --option repositories --json
|
||||
```
|
||||
|
||||
### Latest backups
|
||||
|
||||
All borgmatic actions that accept an `--archive` flag allow you to specify an
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
---
|
||||
title: config show
|
||||
eleventyNavigation:
|
||||
key: config show
|
||||
parent: 🎬 Actions
|
||||
---
|
||||
|
||||
{% include snippet/command-line/sample.md %}
|
||||
|
||||
```
|
||||
{% include borgmatic/command-line/config-show.txt %}
|
||||
```
|
||||
|
||||
|
||||
## Related documentation
|
||||
|
||||
* [Scripting borgmatic](https://torsion.org/borgmatic/how-to/monitor-your-backups/#scripting-borgmatic)
|
||||
@@ -317,3 +317,6 @@ a default location.
|
||||
|
||||
This will output the merged configuration as borgmatic sees it, which can be
|
||||
helpful for understanding how your includes work in practice.
|
||||
|
||||
Also see the [`config show`
|
||||
action](https://torsion.org/borgmatic/reference/command-line/actions/config-show/).
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "borgmatic"
|
||||
version = "2.1.2"
|
||||
version = "2.1.3.dev0"
|
||||
authors = [
|
||||
{ name="Dan Helfman", email="witten@torsion.org" },
|
||||
]
|
||||
|
||||
@@ -35,7 +35,7 @@ BUILTIN_DATASETS = (
|
||||
'used': '256K',
|
||||
'avail': '23.7M',
|
||||
'refer': '25K',
|
||||
'canmount': 'on',
|
||||
'canmount': 'noauto',
|
||||
'mountpoint': '/e2e/pool/dataset',
|
||||
},
|
||||
)
|
||||
|
||||
@@ -292,6 +292,12 @@ def test_render_configuration_converts_configuration_to_yaml_string():
|
||||
assert yaml_string == 'foo: bar\n'
|
||||
|
||||
|
||||
def test_render_configuration_strips_ruamel_yaml_end_of_document_marker():
|
||||
yaml_string = module.render_configuration(33)
|
||||
|
||||
assert yaml_string == '33\n'
|
||||
|
||||
|
||||
def test_write_configuration_does_not_raise():
|
||||
flexmock(os.path).should_receive('exists').and_return(False)
|
||||
flexmock(os).should_receive('makedirs')
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
from flexmock import flexmock
|
||||
|
||||
import borgmatic.logger
|
||||
from borgmatic.actions.config import show as module
|
||||
|
||||
|
||||
def test_run_show_with_single_configuration_file_does_not_separate_output():
|
||||
log_lines = []
|
||||
borgmatic.logger.add_custom_log_levels()
|
||||
|
||||
def fake_logger_answer(message):
|
||||
log_lines.append(message)
|
||||
|
||||
flexmock(module.logger).should_receive('answer').replace_with(fake_logger_answer)
|
||||
show_arguments = flexmock(option=None, json=False)
|
||||
flexmock(module.borgmatic.config.generate).should_receive('render_configuration').and_return(
|
||||
'output'
|
||||
)
|
||||
|
||||
module.run_show(show_arguments, configs={'test.yaml': {}})
|
||||
|
||||
assert log_lines == ['output']
|
||||
|
||||
|
||||
def test_run_show_with_multiple_configuration_files_separates_output():
|
||||
log_lines = []
|
||||
borgmatic.logger.add_custom_log_levels()
|
||||
|
||||
def fake_logger_answer(message):
|
||||
log_lines.append(message)
|
||||
|
||||
flexmock(module.logger).should_receive('answer').replace_with(fake_logger_answer)
|
||||
show_arguments = flexmock(option=None, json=False)
|
||||
flexmock(module.borgmatic.config.generate).should_receive('render_configuration').and_return(
|
||||
'output'
|
||||
).and_return('other')
|
||||
|
||||
module.run_show(show_arguments, configs={'test.yaml': {}, 'other.yaml': {}})
|
||||
|
||||
assert log_lines == ['---', 'output', '---', 'other']
|
||||
|
||||
|
||||
def test_run_show_with_option_limits_output():
|
||||
log_lines = []
|
||||
borgmatic.logger.add_custom_log_levels()
|
||||
|
||||
def fake_logger_answer(message):
|
||||
log_lines.append(message)
|
||||
|
||||
flexmock(module.logger).should_receive('answer').replace_with(fake_logger_answer)
|
||||
show_arguments = flexmock(option='foo', json=False)
|
||||
flexmock(module.borgmatic.config.generate).should_receive('render_configuration').with_args(
|
||||
33
|
||||
).and_return('33')
|
||||
flexmock(module.borgmatic.config.generate).should_receive('render_configuration').with_args(
|
||||
None
|
||||
).and_return('null')
|
||||
|
||||
module.run_show(show_arguments, configs={'test.yaml': {'foo': 33, 'bar': 44}, 'other.yaml': {}})
|
||||
|
||||
assert log_lines == ['---', '33', '---', 'null']
|
||||
|
||||
|
||||
def test_run_show_with_json_outputs_json():
|
||||
flexmock(borgmatic.logger).should_receive('add_custom_log_levels')
|
||||
show_arguments = flexmock(option=None, json=True)
|
||||
flexmock(module.sys.stdout).should_receive('write').with_args(
|
||||
'[{"foo": 33}, {"bar": 44}]'
|
||||
).once()
|
||||
|
||||
module.run_show(show_arguments, configs={'test.yaml': {'foo': 33}, 'other.yaml': {'bar': 44}})
|
||||
|
||||
|
||||
def test_run_show_with_json_and_option_limits_json():
|
||||
flexmock(borgmatic.logger).should_receive('add_custom_log_levels')
|
||||
show_arguments = flexmock(option='foo', json=True)
|
||||
flexmock(module.sys.stdout).should_receive('write').with_args('[33, null]').once()
|
||||
|
||||
module.run_show(show_arguments, configs={'test.yaml': {'foo': 33}, 'other.yaml': {'bar': 44}})
|
||||
@@ -5,13 +5,13 @@ from borgmatic.actions.config import validate as module
|
||||
|
||||
def test_run_validate_does_not_raise():
|
||||
validate_arguments = flexmock(show=False)
|
||||
flexmock(module.borgmatic.config.generate).should_receive('render_configuration')
|
||||
flexmock(module.borgmatic.config.generate).should_receive('render_configuration').and_return('')
|
||||
|
||||
module.run_validate(validate_arguments, flexmock())
|
||||
|
||||
|
||||
def test_run_validate_with_show_does_not_raise():
|
||||
validate_arguments = flexmock(show=True)
|
||||
flexmock(module.borgmatic.config.generate).should_receive('render_configuration')
|
||||
flexmock(module.borgmatic.config.generate).should_receive('render_configuration').and_return('')
|
||||
|
||||
module.run_validate(validate_arguments, {'test.yaml': flexmock(), 'other.yaml': flexmock()})
|
||||
|
||||
@@ -1061,8 +1061,9 @@ def test_compare_spot_check_hashes_returns_paths_having_failing_hashes():
|
||||
'hash2 /bar',
|
||||
)
|
||||
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
|
||||
{'xxh64': 'hash1', 'path': 'foo'},
|
||||
{'xxh64': 'nothash2', 'path': 'bar'},
|
||||
{'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''},
|
||||
{},
|
||||
{'xxh64': 'nothash2', 'path': 'bar', 'linktarget': ''},
|
||||
)
|
||||
|
||||
assert module.compare_spot_check_hashes(
|
||||
@@ -1104,8 +1105,8 @@ def test_compare_spot_check_hashes_handles_weird_backslashed_hashes_from_xxh64su
|
||||
'\\hash2 /bar',
|
||||
)
|
||||
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
|
||||
{'xxh64': 'hash1', 'path': 'foo'},
|
||||
{'xxh64': 'nothash2', 'path': 'bar'},
|
||||
{'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''},
|
||||
{'xxh64': 'nothash2', 'path': 'bar', 'linktarget': ''},
|
||||
)
|
||||
|
||||
assert module.compare_spot_check_hashes(
|
||||
@@ -1147,8 +1148,8 @@ def test_compare_spot_check_hashes_handles_incorrect_path_names_from_xxh64sum():
|
||||
'hash2 /bar/wrong/path',
|
||||
)
|
||||
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
|
||||
{'xxh64': 'hash1', 'path': 'foo'},
|
||||
{'xxh64': 'nothash2', 'path': 'bar'},
|
||||
{'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''},
|
||||
{'xxh64': 'nothash2', 'path': 'bar', 'linktarget': ''},
|
||||
)
|
||||
|
||||
assert module.compare_spot_check_hashes(
|
||||
@@ -1200,8 +1201,8 @@ def test_compare_spot_check_hashes_with_xxh64sum_failure_falls_back_to_individua
|
||||
).once()
|
||||
|
||||
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
|
||||
{'xxh64': 'hash1', 'path': 'foo'},
|
||||
{'xxh64': 'hash2', 'path': 'bar'},
|
||||
{'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''},
|
||||
{'xxh64': 'hash2', 'path': 'bar', 'linktarget': ''},
|
||||
)
|
||||
|
||||
assert module.compare_spot_check_hashes(
|
||||
@@ -1243,8 +1244,8 @@ def test_compare_spot_check_hashes_returns_relative_paths_having_failing_hashes(
|
||||
'hash2 bar',
|
||||
)
|
||||
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
|
||||
{'xxh64': 'hash1', 'path': 'foo'},
|
||||
{'xxh64': 'nothash2', 'path': 'bar'},
|
||||
{'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''},
|
||||
{'xxh64': 'nothash2', 'path': 'bar', 'linktarget': ''},
|
||||
)
|
||||
|
||||
assert module.compare_spot_check_hashes(
|
||||
@@ -1286,8 +1287,8 @@ def test_compare_spot_check_hashes_handles_data_sample_percentage_above_100():
|
||||
'hash2 /bar',
|
||||
)
|
||||
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
|
||||
{'xxh64': 'nothash1', 'path': 'foo'},
|
||||
{'xxh64': 'nothash2', 'path': 'bar'},
|
||||
{'xxh64': 'nothash1', 'path': 'foo', 'linktarget': ''},
|
||||
{'xxh64': 'nothash2', 'path': 'bar', 'linktarget': ''},
|
||||
)
|
||||
|
||||
assert module.compare_spot_check_hashes(
|
||||
@@ -1329,8 +1330,8 @@ def test_compare_spot_check_hashes_uses_xxh64sum_command_option():
|
||||
working_directory=None,
|
||||
).and_yield('hash1 /foo', 'hash2 /bar')
|
||||
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
|
||||
{'xxh64': 'hash1', 'path': 'foo'},
|
||||
{'xxh64': 'nothash2', 'path': 'bar'},
|
||||
{'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''},
|
||||
{'xxh64': 'nothash2', 'path': 'bar', 'linktarget': ''},
|
||||
)
|
||||
|
||||
assert module.compare_spot_check_hashes(
|
||||
@@ -1369,7 +1370,7 @@ def test_compare_spot_check_hashes_considers_path_missing_from_archive_as_not_ma
|
||||
'hash2 /bar',
|
||||
)
|
||||
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
|
||||
{'xxh64': 'hash1', 'path': 'foo'}
|
||||
{'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''}
|
||||
)
|
||||
|
||||
assert module.compare_spot_check_hashes(
|
||||
@@ -1391,6 +1392,50 @@ def test_compare_spot_check_hashes_considers_path_missing_from_archive_as_not_ma
|
||||
) == ('/bar',)
|
||||
|
||||
|
||||
def test_compare_spot_check_hashes_skips_hardlink_path_in_archive():
|
||||
flexmock(module.random).should_receive('SystemRandom').and_return(
|
||||
flexmock(sample=lambda population, count: population[:count]),
|
||||
)
|
||||
flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return(
|
||||
None,
|
||||
)
|
||||
flexmock(module.os.path).should_receive('exists').and_return(True)
|
||||
flexmock(module.os.path).should_receive('islink').and_return(False)
|
||||
flexmock(module.borgmatic.execute).should_receive(
|
||||
'execute_command_and_capture_output',
|
||||
).with_args(('xxh64sum', '/foo', '/bar', '/link'), working_directory=None).and_yield(
|
||||
'hash1 /foo',
|
||||
'hash2 /bar',
|
||||
'hash1 /link',
|
||||
)
|
||||
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
|
||||
{'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''},
|
||||
{'xxh64': 'hash2', 'path': 'bar', 'linktarget': ''},
|
||||
{'xxh64': '', 'path': 'link', 'linktarget': 'foo'},
|
||||
)
|
||||
|
||||
assert (
|
||||
module.compare_spot_check_hashes(
|
||||
repository={'path': 'repo'},
|
||||
archive='archive',
|
||||
config={
|
||||
'checks': [
|
||||
{
|
||||
'name': 'spot',
|
||||
'data_sample_percentage': 100,
|
||||
},
|
||||
],
|
||||
},
|
||||
local_borg_version=flexmock(),
|
||||
global_arguments=flexmock(),
|
||||
local_path=flexmock(),
|
||||
remote_path=flexmock(),
|
||||
source_paths=('/foo', '/bar', '/link'),
|
||||
)
|
||||
== ()
|
||||
)
|
||||
|
||||
|
||||
def test_compare_spot_check_hashes_considers_symlink_path_as_not_matching():
|
||||
flexmock(module.random).should_receive('SystemRandom').and_return(
|
||||
flexmock(sample=lambda population, count: population[:count]),
|
||||
@@ -1405,8 +1450,8 @@ def test_compare_spot_check_hashes_considers_symlink_path_as_not_matching():
|
||||
'execute_command_and_capture_output',
|
||||
).with_args(('xxh64sum', '/foo'), working_directory=None).and_yield('hash1 /foo')
|
||||
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
|
||||
{'xxh64': 'hash1', 'path': 'foo'},
|
||||
{'xxh64': 'hash2', 'path': 'bar'},
|
||||
{'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''},
|
||||
{'xxh64': 'hash2', 'path': 'bar', 'linktarget': ''},
|
||||
)
|
||||
|
||||
assert module.compare_spot_check_hashes(
|
||||
@@ -1442,8 +1487,8 @@ def test_compare_spot_check_hashes_considers_non_existent_path_as_not_matching()
|
||||
'execute_command_and_capture_output',
|
||||
).with_args(('xxh64sum', '/foo'), working_directory=None).and_yield('hash1 /foo')
|
||||
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
|
||||
{'xxh64': 'hash1', 'path': 'foo'},
|
||||
{'xxh64': 'hash2', 'path': 'bar'},
|
||||
{'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''},
|
||||
{'xxh64': 'hash2', 'path': 'bar', 'linktarget': ''},
|
||||
)
|
||||
|
||||
assert module.compare_spot_check_hashes(
|
||||
@@ -1488,11 +1533,11 @@ def test_compare_spot_check_hashes_with_too_many_paths_feeds_them_to_commands_in
|
||||
'hash4 /quux',
|
||||
)
|
||||
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
|
||||
{'xxh64': 'hash1', 'path': 'foo'},
|
||||
{'xxh64': 'hash2', 'path': 'bar'},
|
||||
{'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''},
|
||||
{'xxh64': 'hash2', 'path': 'bar', 'linktarget': ''},
|
||||
).and_yield(
|
||||
{'xxh64': 'hash3', 'path': 'baz'},
|
||||
{'xxh64': 'nothash4', 'path': 'quux'},
|
||||
{'xxh64': 'hash3', 'path': 'baz', 'linktarget': ''},
|
||||
{'xxh64': 'nothash4', 'path': 'quux', 'linktarget': ''},
|
||||
)
|
||||
|
||||
assert module.compare_spot_check_hashes(
|
||||
@@ -1535,8 +1580,8 @@ def test_compare_spot_check_hashes_uses_working_directory_to_access_source_paths
|
||||
'hash2 bar',
|
||||
)
|
||||
flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield(
|
||||
{'xxh64': 'hash1', 'path': 'foo'},
|
||||
{'xxh64': 'nothash2', 'path': 'bar'},
|
||||
{'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''},
|
||||
{'xxh64': 'nothash2', 'path': 'bar', 'linktarget': ''},
|
||||
)
|
||||
|
||||
assert module.compare_spot_check_hashes(
|
||||
|
||||
@@ -2058,6 +2058,43 @@ def test_collect_highlander_action_summary_logs_error_on_run_validate_failure():
|
||||
assert {log.levelno for log in logs} == {logging.CRITICAL}
|
||||
|
||||
|
||||
def test_collect_highlander_action_summary_logs_nothing_additional_for_success_with_show():
|
||||
flexmock(module.borgmatic.actions.config.show).should_receive('run_show')
|
||||
arguments = {
|
||||
'show': flexmock(),
|
||||
'global': flexmock(),
|
||||
}
|
||||
|
||||
logs = tuple(
|
||||
module.collect_highlander_action_summary_logs(
|
||||
{'test.yaml': {}},
|
||||
arguments=arguments,
|
||||
configuration_parse_errors=False,
|
||||
),
|
||||
)
|
||||
assert not logs
|
||||
|
||||
|
||||
def test_collect_highlander_action_summary_logs_error_on_run_show_failure():
|
||||
flexmock(module.borgmatic.actions.config.show).should_receive('run_show').and_raise(
|
||||
ValueError,
|
||||
)
|
||||
arguments = {
|
||||
'show': flexmock(),
|
||||
'global': flexmock(),
|
||||
}
|
||||
|
||||
logs = tuple(
|
||||
module.collect_highlander_action_summary_logs(
|
||||
{'test.yaml': {}},
|
||||
arguments=arguments,
|
||||
configuration_parse_errors=False,
|
||||
),
|
||||
)
|
||||
|
||||
assert {log.levelno for log in logs} == {logging.CRITICAL}
|
||||
|
||||
|
||||
def test_collect_configuration_run_summary_logs_info_for_success():
|
||||
flexmock(module.validate).should_receive('guard_configuration_contains_repository')
|
||||
flexmock(module.command).should_receive('filter_hooks').with_args(
|
||||
|
||||
@@ -1,13 +1,55 @@
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
from flexmock import flexmock
|
||||
|
||||
from borgmatic.hooks.data_source import bootstrap as module
|
||||
|
||||
|
||||
def test_dump_data_sources_creates_manifest_file():
|
||||
flexmock(module.os).should_receive('makedirs')
|
||||
def test_resolve_config_path_symlinks_passes_through_non_symlink():
|
||||
flexmock(module.os.path).should_receive('abspath').replace_with(lambda path: path)
|
||||
flexmock(module.os.path).should_receive('islink').and_return(False)
|
||||
|
||||
assert tuple(module.resolve_config_path_symlinks('test.yaml')) == ('test.yaml',)
|
||||
|
||||
|
||||
def test_resolve_config_path_symlinks_follows_each_symlink():
|
||||
flexmock(module.os.path).should_receive('abspath').replace_with(lambda path: path)
|
||||
flexmock(module.os.path).should_receive('islink').with_args('test.yaml').and_return(True)
|
||||
flexmock(module.os.path).should_receive('islink').with_args('dest1.yaml').and_return(True)
|
||||
flexmock(module.os.path).should_receive('islink').with_args('dest2.yaml').and_return(False)
|
||||
flexmock(module.os).should_receive('readlink').with_args('test.yaml').and_return('dest1.yaml')
|
||||
flexmock(module.os).should_receive('readlink').with_args('dest1.yaml').and_return('dest2.yaml')
|
||||
flexmock(module.os).should_receive('readlink').with_args('dest2.yaml').never()
|
||||
|
||||
assert tuple(module.resolve_config_path_symlinks('test.yaml')) == (
|
||||
'test.yaml',
|
||||
'dest1.yaml',
|
||||
'dest2.yaml',
|
||||
)
|
||||
|
||||
|
||||
def test_resolve_config_path_symlinks_with_too_many_symlinks_raises():
|
||||
flexmock(module).MAXIMUM_CONFIG_SYMLINKS_TO_FOLLOW = 2
|
||||
flexmock(module.os.path).should_receive('abspath').replace_with(lambda path: path)
|
||||
flexmock(module.os.path).should_receive('islink').with_args('test.yaml').and_return(True)
|
||||
flexmock(module.os.path).should_receive('islink').with_args('dest1.yaml').and_return(True)
|
||||
flexmock(module.os.path).should_receive('islink').with_args('dest2.yaml').and_return(True)
|
||||
flexmock(module.os.path).should_receive('islink').with_args('dest3.yaml').never()
|
||||
flexmock(module.os).should_receive('readlink').with_args('test.yaml').and_return('dest1.yaml')
|
||||
flexmock(module.os).should_receive('readlink').with_args('dest1.yaml').and_return('dest2.yaml')
|
||||
flexmock(module.os).should_receive('readlink').with_args('dest2.yaml').and_return('dest3.yaml')
|
||||
flexmock(module.os).should_receive('readlink').with_args('dest3.yaml').never()
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
assert tuple(module.resolve_config_path_symlinks('test.yaml'))
|
||||
|
||||
|
||||
def test_dump_data_sources_creates_manifest_file():
|
||||
flexmock(module).should_receive('resolve_config_path_symlinks').and_yield(
|
||||
'test.yaml', 'linkdest.yaml'
|
||||
)
|
||||
flexmock(module.os).should_receive('makedirs')
|
||||
flexmock(module.importlib.metadata).should_receive('version').and_return('1.0.0')
|
||||
manifest_file = flexmock(
|
||||
__enter__=lambda *args: flexmock(write=lambda *args: None, close=lambda *args: None),
|
||||
@@ -19,7 +61,7 @@ def test_dump_data_sources_creates_manifest_file():
|
||||
encoding='utf-8',
|
||||
).and_return(manifest_file)
|
||||
flexmock(module.json).should_receive('dump').with_args(
|
||||
{'borgmatic_version': '1.0.0', 'config_paths': ('test.yaml',)},
|
||||
{'borgmatic_version': '1.0.0', 'config_paths': ('test.yaml', 'linkdest.yaml')},
|
||||
manifest_file,
|
||||
).once()
|
||||
flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
|
||||
@@ -34,6 +76,12 @@ def test_dump_data_sources_creates_manifest_file():
|
||||
'test.yaml', source=module.borgmatic.borg.pattern.Pattern_source.HOOK
|
||||
),
|
||||
).once()
|
||||
flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args(
|
||||
object,
|
||||
module.borgmatic.borg.pattern.Pattern(
|
||||
'linkdest.yaml', source=module.borgmatic.borg.pattern.Pattern_source.HOOK
|
||||
),
|
||||
).once()
|
||||
|
||||
module.dump_data_sources(
|
||||
hook_config=None,
|
||||
@@ -46,6 +94,7 @@ def test_dump_data_sources_creates_manifest_file():
|
||||
|
||||
|
||||
def test_dump_data_sources_with_store_config_files_false_does_not_create_manifest_file():
|
||||
flexmock(module).should_receive('resolve_config_path_symlinks').and_yield('test.yaml')
|
||||
flexmock(module.os).should_receive('makedirs').never()
|
||||
flexmock(module.json).should_receive('dump').never()
|
||||
flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').never()
|
||||
@@ -62,6 +111,7 @@ def test_dump_data_sources_with_store_config_files_false_does_not_create_manifes
|
||||
|
||||
|
||||
def test_dump_data_sources_with_dry_run_does_not_create_manifest_file():
|
||||
flexmock(module).should_receive('resolve_config_path_symlinks').and_yield('test.yaml')
|
||||
flexmock(module.os).should_receive('makedirs').never()
|
||||
flexmock(module.json).should_receive('dump').never()
|
||||
flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').never()
|
||||
|
||||
Reference in New Issue
Block a user