From d532fc0f88f0999ed473eda6d58aef424c4f0286 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sat, 14 Feb 2026 21:30:53 -0800 Subject: [PATCH 1/7] Add a "config show" action to display computed borgmatic configuration as YAML or JSON (#1218). --- NEWS | 6 ++ borgmatic/actions/config/show.py | 44 ++++++++++ borgmatic/commands/arguments.py | 23 +++++ borgmatic/commands/borgmatic.py | 87 +++++++++---------- borgmatic/config/generate.py | 13 ++- docs/Dockerfile | 2 +- docs/how-to/monitor-your-backups.md | 21 +++++ .../command-line/actions/config-show.md | 17 ++++ docs/reference/configuration/includes.md | 3 + pyproject.toml | 2 +- tests/integration/config/test_generate.py | 6 ++ tests/unit/actions/config/test_show.py | 79 +++++++++++++++++ tests/unit/actions/config/test_validate.py | 4 +- tests/unit/commands/test_borgmatic.py | 37 ++++++++ 14 files changed, 292 insertions(+), 52 deletions(-) create mode 100644 borgmatic/actions/config/show.py create mode 100644 docs/reference/command-line/actions/config-show.md create mode 100644 tests/unit/actions/config/test_show.py diff --git a/NEWS b/NEWS index 51724748..1d854410 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,9 @@ +2.1.3.dev0 + * #1218: Add a "config show" action to display computed borgmatic configuration as YAML or JSON, + handy for fetching borgmatic configuration from external scripts. See the documentation for more + information: + https://torsion.org/borgmatic/reference/command-line/actions/config-show/ + 2.1.2 * #1231: If a source file is deleted during a "spot" check, consider the file as non-matching and move on instead of immediately failing the entire check. diff --git a/borgmatic/actions/config/show.py b/borgmatic/actions/config/show.py new file mode 100644 index 00000000..bfa05b0d --- /dev/null +++ b/borgmatic/actions/config/show.py @@ -0,0 +1,44 @@ +import json +import logging +import sys + +import borgmatic.config.generate +import borgmatic.logger + +logger = logging.getLogger(__name__) + + +def run_show(show_arguments, configs): + ''' + Given the show arguments as an argparse.Namespace instance and a dict of configuration filename + to corresponding parsed configuration, run the "show" action. That consists of rendering and + logging the computed configuration as YAML, separating the configuration for each file with + "---". + + If show_arguments.option is set, limit the results to the value of that single option. If + show_arguments.json is True, render the results as JSON with one array element per configuration + file. + ''' + borgmatic.logger.add_custom_log_levels() + + if show_arguments.json: + sys.stdout.write( + json.dumps( + [ + config.get(show_arguments.option) if show_arguments.option else config + for config in configs.values() + ] + ) + ) + + return + + for config in configs.values(): + if len(configs) > 1: + logger.answer('---') + + logger.answer( + borgmatic.config.generate.render_configuration( + config.get(show_arguments.option) if show_arguments.option else config + ).rstrip() + ) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 58c3a632..73a71556 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1261,6 +1261,29 @@ def make_parsers(schema, unparsed_arguments): # noqa: PLR0915 help='Show this help message and exit', ) + config_show_parser = config_parsers.add_parser( + 'show', + help='Show the computed configuration for each file specified with --config (see borgmatic --help)', + description='Show the computed configuration for each file specified with --config (see borgmatic --help)', + add_help=False, + ) + config_show_group = config_show_parser.add_argument_group('config show arguments') + config_show_group.add_argument( + '--option', + help='Show the value of a single named configuration option instead of the entire configuration', + ) + config_show_group.add_argument( + '--json', + action='store_true', + help='Show the configuration as JSON with one array element per configuration file', + ) + config_show_group.add_argument( + '-h', + '--help', + action='help', + help='Show this help message and exit', + ) + export_tar_parser = action_parsers.add_parser( 'export-tar', aliases=ACTION_ALIASES['export-tar'], diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 2954ac72..7ddaea8c 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -17,6 +17,7 @@ import borgmatic.actions.check import borgmatic.actions.compact import borgmatic.actions.config.bootstrap import borgmatic.actions.config.generate +import borgmatic.actions.config.show import borgmatic.actions.config.validate import borgmatic.actions.create import borgmatic.actions.delete @@ -810,18 +811,18 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par ''' add_custom_log_levels() - if 'bootstrap' in arguments: - try: - # No configuration file is needed for bootstrap. - local_borg_version = borg_version.local_borg_version( - {}, - arguments['bootstrap'].local_path, - ) - except (OSError, CalledProcessError, ValueError) as error: - yield from log_error_records('Error getting local Borg version', error) - return + try: + if 'bootstrap' in arguments: + try: + # No configuration file is needed for bootstrap. + local_borg_version = borg_version.local_borg_version( + {}, + arguments['bootstrap'].local_path, + ) + except (OSError, CalledProcessError, ValueError) as error: + yield from log_error_records('Error getting local Borg version', error) + return - try: borgmatic.actions.config.bootstrap.run_bootstrap( arguments['bootstrap'], arguments['global'], @@ -835,17 +836,10 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par name=logger.name, ), ) - except ( - CalledProcessError, - ValueError, - OSError, - ) as error: - yield from log_error_records(error) - return + return - if 'generate' in arguments: - try: + if 'generate' in arguments: borgmatic.actions.config.generate.run_generate( arguments['generate'], arguments['global'], @@ -858,29 +852,22 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par name=logger.name, ), ) - except ( - CalledProcessError, - ValueError, - OSError, - ) as error: - yield from log_error_records(error) - - return - - if 'validate' in arguments: - if configuration_parse_errors: - yield logging.makeLogRecord( - dict( - levelno=logging.CRITICAL, - levelname='CRITICAL', - msg='Configuration validation failed', - name=logger.name, - ), - ) return - try: + if 'validate' in arguments: + if configuration_parse_errors: + yield logging.makeLogRecord( + dict( + levelno=logging.CRITICAL, + levelname='CRITICAL', + msg='Configuration validation failed', + name=logger.name, + ), + ) + + return + borgmatic.actions.config.validate.run_validate(arguments['validate'], configs) yield logging.makeLogRecord( @@ -891,14 +878,20 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par name=logger.name, ), ) - except ( - CalledProcessError, - ValueError, - OSError, - ) as error: - yield from log_error_records(error) - return + return + + if 'show' in arguments: + borgmatic.actions.config.show.run_show(arguments['show'], configs) + + return + + except ( + CalledProcessError, + ValueError, + OSError, + ) as error: + yield from log_error_records(error) def collect_configuration_run_summary_logs(configs, config_paths, arguments, log_file_path): # noqa: PLR0912 diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index 76d4a167..4d3cda78 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -153,6 +153,9 @@ def transform_optional_configuration(rendered_config, comment_out=True): return '\n'.join(lines) +RUAMEL_YAML_END_OF_DOCUMENT_MARKER = '...\n' + + def render_configuration(config): ''' Given a config data structure of nested OrderedDicts, render the config as YAML and return it. @@ -160,7 +163,15 @@ def render_configuration(config): dumper = ruamel.yaml.YAML(typ='rt') dumper.indent(mapping=INDENT, sequence=INDENT + SEQUENCE_INDENT, offset=INDENT) rendered = io.StringIO() - dumper.dump(config, rendered) + dumper.dump( + config, + rendered, + # Dumping certain values (integers, for instance) causes ruamel.yaml to append an + # end-of-document "..." marker. Strip it. + transform=lambda dumped: dumped[: -len(RUAMEL_YAML_END_OF_DOCUMENT_MARKER)] + if dumped.endswith(RUAMEL_YAML_END_OF_DOCUMENT_MARKER) + else dumped, + ) return rendered.getvalue() diff --git a/docs/Dockerfile b/docs/Dockerfile index 5463b98d..a58c92c0 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -5,7 +5,7 @@ RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib RUN pip install --break-system-packages --no-cache /app && borgmatic config generate && borgmatic config generate --destination /etc/borgmatic --split && chmod +r /etc/borgmatic/*.yaml RUN mkdir /command-line \ && borgmatic --help > /command-line/global.txt \ - && for action in repo-create transfer create prune compact check delete extract config "config bootstrap" "config generate" "config validate" export-tar mount umount repo-delete restore repo-list list repo-info info break-lock "key export" "key import" "key change-passphrase" recreate borg; do \ + && for action in repo-create transfer create prune compact check delete extract config "config bootstrap" "config generate" "config validate" "config show" export-tar mount umount repo-delete restore repo-list list repo-info info break-lock "key export" "key import" "key change-passphrase" recreate borg; do \ borgmatic $action --help > /command-line/${action/ /-}.txt; done RUN /app/docs/fetch-contributors >> /contributors.html diff --git a/docs/how-to/monitor-your-backups.md b/docs/how-to/monitor-your-backups.md index 763162fc..963e07d2 100644 --- a/docs/how-to/monitor-your-backups.md +++ b/docs/how-to/monitor-your-backups.md @@ -64,6 +64,27 @@ suppressed so as not to interfere with the captured JSON. Also note that JSON output only shows up at the console and not in syslog. +### Getting configuration + +New in version 2.1.3 If you want +to consume borgmatic's computed configuration in your scripts, use the [`config +show` +action](https://torsion.org/borgmatic/reference/command-line/actions/config-show/). +Here's an example: + +```bash +borgmatic config show --json +``` + +That outputs borgmatic's entire configuration as JSON with one array element per +configuration file. + +Or you can ask for the value of a particular option: + +```bash +borgmatic config show --option repositories --json +``` + ### Latest backups All borgmatic actions that accept an `--archive` flag allow you to specify an diff --git a/docs/reference/command-line/actions/config-show.md b/docs/reference/command-line/actions/config-show.md new file mode 100644 index 00000000..12ad4552 --- /dev/null +++ b/docs/reference/command-line/actions/config-show.md @@ -0,0 +1,17 @@ +--- +title: config show +eleventyNavigation: + key: config show + parent: 🎬 Actions +--- + +{% include snippet/command-line/sample.md %} + +``` +{% include borgmatic/command-line/config-show.txt %} +``` + + +## Related documentation + + * [Scripting borgmatic](https://torsion.org/borgmatic/how-to/monitor-your-backups/#scripting-borgmatic) diff --git a/docs/reference/configuration/includes.md b/docs/reference/configuration/includes.md index 1b2691fb..2175fe2e 100644 --- a/docs/reference/configuration/includes.md +++ b/docs/reference/configuration/includes.md @@ -317,3 +317,6 @@ a default location. This will output the merged configuration as borgmatic sees it, which can be helpful for understanding how your includes work in practice. + +Also see the [`config show` +action](https://torsion.org/borgmatic/reference/command-line/actions/config-show/). diff --git a/pyproject.toml b/pyproject.toml index 8c0274a2..88a1d506 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "borgmatic" -version = "2.1.2" +version = "2.1.3.dev0" authors = [ { name="Dan Helfman", email="witten@torsion.org" }, ] diff --git a/tests/integration/config/test_generate.py b/tests/integration/config/test_generate.py index 48fbd502..14b03627 100644 --- a/tests/integration/config/test_generate.py +++ b/tests/integration/config/test_generate.py @@ -292,6 +292,12 @@ def test_render_configuration_converts_configuration_to_yaml_string(): assert yaml_string == 'foo: bar\n' +def test_render_configuration_strips_ruamel_yaml_end_of_document_marker(): + yaml_string = module.render_configuration(33) + + assert yaml_string == '33\n' + + def test_write_configuration_does_not_raise(): flexmock(os.path).should_receive('exists').and_return(False) flexmock(os).should_receive('makedirs') diff --git a/tests/unit/actions/config/test_show.py b/tests/unit/actions/config/test_show.py new file mode 100644 index 00000000..2f12c7ea --- /dev/null +++ b/tests/unit/actions/config/test_show.py @@ -0,0 +1,79 @@ +from flexmock import flexmock + +import borgmatic.logger +from borgmatic.actions.config import show as module + + +def test_run_show_with_single_configuration_file_does_not_separate_output(): + log_lines = [] + borgmatic.logger.add_custom_log_levels() + + def fake_logger_answer(message): + log_lines.append(message) + + flexmock(module.logger).should_receive('answer').replace_with(fake_logger_answer) + show_arguments = flexmock(option=None, json=False) + flexmock(module.borgmatic.config.generate).should_receive('render_configuration').and_return( + 'output' + ) + + module.run_show(show_arguments, configs={'test.yaml': {}}) + + assert log_lines == ['output'] + + +def test_run_show_with_multiple_configuration_files_separates_output(): + log_lines = [] + borgmatic.logger.add_custom_log_levels() + + def fake_logger_answer(message): + log_lines.append(message) + + flexmock(module.logger).should_receive('answer').replace_with(fake_logger_answer) + show_arguments = flexmock(option=None, json=False) + flexmock(module.borgmatic.config.generate).should_receive('render_configuration').and_return( + 'output' + ).and_return('other') + + module.run_show(show_arguments, configs={'test.yaml': {}, 'other.yaml': {}}) + + assert log_lines == ['---', 'output', '---', 'other'] + + +def test_run_show_with_option_limits_output(): + log_lines = [] + borgmatic.logger.add_custom_log_levels() + + def fake_logger_answer(message): + log_lines.append(message) + + flexmock(module.logger).should_receive('answer').replace_with(fake_logger_answer) + show_arguments = flexmock(option='foo', json=False) + flexmock(module.borgmatic.config.generate).should_receive('render_configuration').with_args( + 33 + ).and_return('33') + flexmock(module.borgmatic.config.generate).should_receive('render_configuration').with_args( + None + ).and_return('null') + + module.run_show(show_arguments, configs={'test.yaml': {'foo': 33, 'bar': 44}, 'other.yaml': {}}) + + assert log_lines == ['---', '33', '---', 'null'] + + +def test_run_show_with_json_outputs_json(): + flexmock(borgmatic.logger).should_receive('add_custom_log_levels') + show_arguments = flexmock(option=None, json=True) + flexmock(module.sys.stdout).should_receive('write').with_args( + '[{"foo": 33}, {"bar": 44}]' + ).once() + + module.run_show(show_arguments, configs={'test.yaml': {'foo': 33}, 'other.yaml': {'bar': 44}}) + + +def test_run_show_with_json_and_option_limits_json(): + flexmock(borgmatic.logger).should_receive('add_custom_log_levels') + show_arguments = flexmock(option='foo', json=True) + flexmock(module.sys.stdout).should_receive('write').with_args('[33, null]').once() + + module.run_show(show_arguments, configs={'test.yaml': {'foo': 33}, 'other.yaml': {'bar': 44}}) diff --git a/tests/unit/actions/config/test_validate.py b/tests/unit/actions/config/test_validate.py index 862d1bfa..f24db6c4 100644 --- a/tests/unit/actions/config/test_validate.py +++ b/tests/unit/actions/config/test_validate.py @@ -5,13 +5,13 @@ from borgmatic.actions.config import validate as module def test_run_validate_does_not_raise(): validate_arguments = flexmock(show=False) - flexmock(module.borgmatic.config.generate).should_receive('render_configuration') + flexmock(module.borgmatic.config.generate).should_receive('render_configuration').and_return('') module.run_validate(validate_arguments, flexmock()) def test_run_validate_with_show_does_not_raise(): validate_arguments = flexmock(show=True) - flexmock(module.borgmatic.config.generate).should_receive('render_configuration') + flexmock(module.borgmatic.config.generate).should_receive('render_configuration').and_return('') module.run_validate(validate_arguments, {'test.yaml': flexmock(), 'other.yaml': flexmock()}) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 887f1631..7caa1596 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -2058,6 +2058,43 @@ def test_collect_highlander_action_summary_logs_error_on_run_validate_failure(): assert {log.levelno for log in logs} == {logging.CRITICAL} +def test_collect_highlander_action_summary_logs_nothing_additional_for_success_with_show(): + flexmock(module.borgmatic.actions.config.show).should_receive('run_show') + arguments = { + 'show': flexmock(), + 'global': flexmock(), + } + + logs = tuple( + module.collect_highlander_action_summary_logs( + {'test.yaml': {}}, + arguments=arguments, + configuration_parse_errors=False, + ), + ) + assert not logs + + +def test_collect_highlander_action_summary_logs_error_on_run_show_failure(): + flexmock(module.borgmatic.actions.config.show).should_receive('run_show').and_raise( + ValueError, + ) + arguments = { + 'show': flexmock(), + 'global': flexmock(), + } + + logs = tuple( + module.collect_highlander_action_summary_logs( + {'test.yaml': {}}, + arguments=arguments, + configuration_parse_errors=False, + ), + ) + + assert {log.levelno for log in logs} == {logging.CRITICAL} + + def test_collect_configuration_run_summary_logs_info_for_success(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') flexmock(module.command).should_receive('filter_hooks').with_args( From 55375cc9e758a55e33774d4968ffad0ca075dfdb Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 15 Feb 2026 11:06:34 -0800 Subject: [PATCH 2/7] Fix the ZFS hook to support datasets with a "canmount" property of "noauto" (#1269). --- NEWS | 1 + borgmatic/hooks/data_source/zfs.py | 2 +- tests/end-to-end/commands/fake_zfs.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 1d854410..b6e204b0 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,7 @@ handy for fetching borgmatic configuration from external scripts. See the documentation for more information: https://torsion.org/borgmatic/reference/command-line/actions/config-show/ + * #1269: Fix the ZFS hook to support datasets with a "canmount" property of "noauto". 2.1.2 * #1231: If a source file is deleted during a "spot" check, consider the file as non-matching diff --git a/borgmatic/hooks/data_source/zfs.py b/borgmatic/hooks/data_source/zfs.py index 40ae25d4..f8eb840a 100644 --- a/borgmatic/hooks/data_source/zfs.py +++ b/borgmatic/hooks/data_source/zfs.py @@ -71,7 +71,7 @@ def get_datasets_to_backup(zfs_command, patterns): ) # Skip datasets that are marked "canmount=off", because mounting their snapshots will # result in completely empty mount points—thereby preventing us from backing them up. - if can_mount == 'on' + if can_mount != 'off' ), key=lambda dataset: dataset.mount_point, reverse=True, diff --git a/tests/end-to-end/commands/fake_zfs.py b/tests/end-to-end/commands/fake_zfs.py index 4ef3f498..cceaf59d 100644 --- a/tests/end-to-end/commands/fake_zfs.py +++ b/tests/end-to-end/commands/fake_zfs.py @@ -35,7 +35,7 @@ BUILTIN_DATASETS = ( 'used': '256K', 'avail': '23.7M', 'refer': '25K', - 'canmount': 'on', + 'canmount': 'noauto', 'mountpoint': '/e2e/pool/dataset', }, ) From 90d18574940bc0557964fe0206831cda3a40874c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 17 Feb 2026 10:16:47 -0800 Subject: [PATCH 3/7] Remove GitHub pull request template, because PRs are disabled there now. --- .github/pull_request_template.md | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md deleted file mode 100644 index 6256e53a..00000000 --- a/.github/pull_request_template.md +++ /dev/null @@ -1,10 +0,0 @@ -## Hold up, GitHub users - -Thanks for your contribution! - -Unfortunately, we don't use GitHub pull requests to manage code contributions to this repository (and GitHub doesn't have any way to disable pull requests entirely). Instead, please see: - -https://torsion.org/borgmatic/#contributing - -... which provides full instructions on how to submit pull requests. You can even use your GitHub account to login. - From c7c2ef048cf61ecd21c63e8b08bfee56cbbfb1cb Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Tue, 17 Feb 2026 11:00:16 -0800 Subject: [PATCH 4/7] Remove GitHub PRs URL from contributor fetching script. --- docs/fetch-contributors | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/fetch-contributors b/docs/fetch-contributors index 7c1e1093..a9734d73 100755 --- a/docs/fetch-contributors +++ b/docs/fetch-contributors @@ -36,7 +36,6 @@ def list_contributing_issues(url): PULLS_API_ENDPOINT_URLS = ( 'https://projects.torsion.org/api/v1/repos/borgmatic-collective/borgmatic/pulls', - 'https://api.github.com/repos/borgmatic-collective/borgmatic/pulls', ) ISSUES_API_ENDPOINT_URL = 'https://projects.torsion.org/api/v1/repos/borgmatic-collective/borgmatic/issues?state=all' RECENT_CONTRIBUTORS_CUTOFF_DAYS = 365 From 67eb48e6433beaab5b7472ceb190219d5b724149 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Thu, 19 Feb 2026 11:11:57 -0800 Subject: [PATCH 5/7] Add a policy about the use of generative AI in the borgmatic codebase. --- AGENTS.md | 28 ++++++++++++++++++++++++++++ NEWS | 2 ++ docs/how-to/develop-on-borgmatic.md | 14 ++++++++++++++ 3 files changed, 44 insertions(+) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..a6f7c7bb --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,28 @@ +# AGENTS.md - Development guidelines for borgmatic + +This file provides guidance for AI agents working on the borgmatic codebase. + +## Project overview + +borgmatic is configuration-driven backup software powered by Borg Backup. It's a +Python project using setuptools. + +Please do not use AI agents to modify this codebase. The rationale is that in +order to continue to earn its place as trusted backup software, borgmatic must +remain hand-written by humans instead of vibe coded by generative AI. + +Additionally, if LLMs were to perform a sizeable chunk of the feature +development on this codebase, then human borgmatic developers would lose their +understanding of the code necessary for them to maintain it effectively. + +Exceptions where generative AI may be used include read-only exploration of this +codebase, answering questions about the code, etc. + +## Architecture notes + +- **main entry point**: `borgmatic.commands.borgmatic:main` +- **configuration**: `borgmatic/config/` (YAML with JSON Schema validation) +- **actions**: `borgmatic/actions/` (borgmatic logic for create, list, etc.) +- **Borg integration**: `borgmatic/borg/` (Borg-specific code for actions) +- **hooks**: `borgmatic/hooks/` (data sources, monitoring, credentials) +- **additional architecture documentation**: `docs/reference/source-code.md` diff --git a/NEWS b/NEWS index b6e204b0..fb29771d 100644 --- a/NEWS +++ b/NEWS @@ -4,6 +4,8 @@ information: https://torsion.org/borgmatic/reference/command-line/actions/config-show/ * #1269: Fix the ZFS hook to support datasets with a "canmount" property of "noauto". + * Add a policy about the use of generative AI in the borgmatic codebase: + https://torsion.org/borgmatic/how-to/develop-on-borgmatic/#use-of-generative-ai 2.1.2 * #1231: If a source file is deleted during a "spot" check, consider the file as non-matching diff --git a/docs/how-to/develop-on-borgmatic.md b/docs/how-to/develop-on-borgmatic.md index ef74841d..762abc36 100644 --- a/docs/how-to/develop-on-borgmatic.md +++ b/docs/how-to/develop-on-borgmatic.md @@ -208,4 +208,18 @@ Setting up Podman is outside the scope of this documentation. But once you install and configure Podman, then `scripts/dev-docs` should automatically use Podman instead of Docker. + +## Use of generative AI + +Please do not use AI agents to modify this codebase. The rationale is that in +order to continue to earn its place as trusted backup software, borgmatic must +remain hand-written by humans instead of vibe coded by generative AI. + +Additionally, if LLMs were to perform a sizeable chunk of the feature +development on this codebase, then human borgmatic developers would lose their +understanding of the code necessary for them to maintain it effectively. + +Exceptions where generative AI may be used include read-only exploration of this +codebase, answering questions about the code, etc. + From 7ca42a8f4f9b13f0300b4263ce844a6eb6afe265 Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Fri, 20 Feb 2026 19:55:52 -0800 Subject: [PATCH 6/7] Follow symlinks when backing up borgmatic configuration files to support the "bootstrap" action (#1270). --- NEWS | 2 + borgmatic/actions/config/bootstrap.py | 2 +- borgmatic/config/collect.py | 1 + borgmatic/hooks/data_source/bootstrap.py | 35 +++++++++++- .../unit/hooks/data_source/test_bootstrap.py | 56 ++++++++++++++++++- 5 files changed, 90 insertions(+), 6 deletions(-) diff --git a/NEWS b/NEWS index fb29771d..285a9093 100644 --- a/NEWS +++ b/NEWS @@ -4,6 +4,8 @@ information: https://torsion.org/borgmatic/reference/command-line/actions/config-show/ * #1269: Fix the ZFS hook to support datasets with a "canmount" property of "noauto". + * #1270: Follow symlinks when backing up borgmatic configuration files to support the "bootstrap" + action. * Add a policy about the use of generative AI in the borgmatic codebase: https://torsion.org/borgmatic/how-to/develop-on-borgmatic/#use-of-generative-ai diff --git a/borgmatic/actions/config/bootstrap.py b/borgmatic/actions/config/bootstrap.py index 5e035a11..a561c7b7 100644 --- a/borgmatic/actions/config/bootstrap.py +++ b/borgmatic/actions/config/bootstrap.py @@ -120,7 +120,7 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version): borgmatic_runtime_directory, ) - logger.info(f"Bootstrapping config paths: {', '.join(manifest_config_paths)}") + logger.info(f"Bootstrapping configuration paths: {', '.join(manifest_config_paths)}") borgmatic.borg.extract.extract_archive( global_arguments.dry_run, diff --git a/borgmatic/config/collect.py b/borgmatic/config/collect.py index 0edf1412..c30e68f1 100644 --- a/borgmatic/config/collect.py +++ b/borgmatic/config/collect.py @@ -47,5 +47,6 @@ def collect_config_filenames(config_paths): for filename in sorted(os.listdir(path)): full_filename = os.path.join(path, filename) matching_filetype = full_filename.endswith(('.yaml', '.yml')) + if matching_filetype and not os.path.isdir(full_filename): yield os.path.abspath(full_filename) diff --git a/borgmatic/hooks/data_source/bootstrap.py b/borgmatic/hooks/data_source/bootstrap.py index 6765d33c..3b227554 100644 --- a/borgmatic/hooks/data_source/bootstrap.py +++ b/borgmatic/hooks/data_source/bootstrap.py @@ -1,6 +1,7 @@ import contextlib import glob import importlib +import itertools import json import logging import os @@ -19,6 +20,29 @@ def use_streaming(hook_config, config): # pragma: no cover return False +MAXIMUM_CONFIG_SYMLINKS_TO_FOLLOW = 10 + + +def resolve_config_path_symlinks(path): + ''' + Given a path, resolve and yield each successive symlink until the final non-symlink target. If + the given path isn't a symlink, then just yield it. + + Raise ValueError if we have to follow too many symlinks without getting to the final target. + ''' + original_path = path + + for _ in range(MAXIMUM_CONFIG_SYMLINKS_TO_FOLLOW): + yield os.path.abspath(path) + + if not os.path.islink(path): + return + + path = os.readlink(path) + + raise ValueError(f'Too many symlinks to follow for configuration path: {original_path}') + + def dump_data_sources( hook_config, config, @@ -34,6 +58,9 @@ def dump_data_sources( the archive. But skip this if the bootstrap store_config_files option is False or if this is a dry run. + If any configuration paths are symlinks, then store each symlink along with any destination + paths as well. + Return an empty sequence, since there are no ongoing dump processes from this hook. ''' if hook_config and hook_config.get('store_config_files') is False: @@ -45,6 +72,10 @@ def dump_data_sources( 'manifest.json', ) + resolved_config_paths = tuple( + itertools.chain.from_iterable(resolve_config_path_symlinks(path) for path in config_paths) + ) + if dry_run: return [] @@ -54,7 +85,7 @@ def dump_data_sources( json.dump( { 'borgmatic_version': importlib.metadata.version('borgmatic'), - 'config_paths': config_paths, + 'config_paths': resolved_config_paths, }, manifest_file, ) @@ -67,7 +98,7 @@ def dump_data_sources( ), ) - for config_path in config_paths: + for config_path in resolved_config_paths: borgmatic.hooks.data_source.config.inject_pattern( patterns, borgmatic.borg.pattern.Pattern( diff --git a/tests/unit/hooks/data_source/test_bootstrap.py b/tests/unit/hooks/data_source/test_bootstrap.py index 6958f0f1..b768bbc6 100644 --- a/tests/unit/hooks/data_source/test_bootstrap.py +++ b/tests/unit/hooks/data_source/test_bootstrap.py @@ -1,13 +1,55 @@ import sys +import pytest from flexmock import flexmock from borgmatic.hooks.data_source import bootstrap as module -def test_dump_data_sources_creates_manifest_file(): - flexmock(module.os).should_receive('makedirs') +def test_resolve_config_path_symlinks_passes_through_non_symlink(): + flexmock(module.os.path).should_receive('abspath').replace_with(lambda path: path) + flexmock(module.os.path).should_receive('islink').and_return(False) + assert tuple(module.resolve_config_path_symlinks('test.yaml')) == ('test.yaml',) + + +def test_resolve_config_path_symlinks_follows_each_symlink(): + flexmock(module.os.path).should_receive('abspath').replace_with(lambda path: path) + flexmock(module.os.path).should_receive('islink').with_args('test.yaml').and_return(True) + flexmock(module.os.path).should_receive('islink').with_args('dest1.yaml').and_return(True) + flexmock(module.os.path).should_receive('islink').with_args('dest2.yaml').and_return(False) + flexmock(module.os).should_receive('readlink').with_args('test.yaml').and_return('dest1.yaml') + flexmock(module.os).should_receive('readlink').with_args('dest1.yaml').and_return('dest2.yaml') + flexmock(module.os).should_receive('readlink').with_args('dest2.yaml').never() + + assert tuple(module.resolve_config_path_symlinks('test.yaml')) == ( + 'test.yaml', + 'dest1.yaml', + 'dest2.yaml', + ) + + +def test_resolve_config_path_symlinks_with_too_many_symlinks_raises(): + flexmock(module).MAXIMUM_CONFIG_SYMLINKS_TO_FOLLOW = 2 + flexmock(module.os.path).should_receive('abspath').replace_with(lambda path: path) + flexmock(module.os.path).should_receive('islink').with_args('test.yaml').and_return(True) + flexmock(module.os.path).should_receive('islink').with_args('dest1.yaml').and_return(True) + flexmock(module.os.path).should_receive('islink').with_args('dest2.yaml').and_return(True) + flexmock(module.os.path).should_receive('islink').with_args('dest3.yaml').never() + flexmock(module.os).should_receive('readlink').with_args('test.yaml').and_return('dest1.yaml') + flexmock(module.os).should_receive('readlink').with_args('dest1.yaml').and_return('dest2.yaml') + flexmock(module.os).should_receive('readlink').with_args('dest2.yaml').and_return('dest3.yaml') + flexmock(module.os).should_receive('readlink').with_args('dest3.yaml').never() + + with pytest.raises(ValueError): + assert tuple(module.resolve_config_path_symlinks('test.yaml')) + + +def test_dump_data_sources_creates_manifest_file(): + flexmock(module).should_receive('resolve_config_path_symlinks').and_yield( + 'test.yaml', 'linkdest.yaml' + ) + flexmock(module.os).should_receive('makedirs') flexmock(module.importlib.metadata).should_receive('version').and_return('1.0.0') manifest_file = flexmock( __enter__=lambda *args: flexmock(write=lambda *args: None, close=lambda *args: None), @@ -19,7 +61,7 @@ def test_dump_data_sources_creates_manifest_file(): encoding='utf-8', ).and_return(manifest_file) flexmock(module.json).should_receive('dump').with_args( - {'borgmatic_version': '1.0.0', 'config_paths': ('test.yaml',)}, + {'borgmatic_version': '1.0.0', 'config_paths': ('test.yaml', 'linkdest.yaml')}, manifest_file, ).once() flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args( @@ -34,6 +76,12 @@ def test_dump_data_sources_creates_manifest_file(): 'test.yaml', source=module.borgmatic.borg.pattern.Pattern_source.HOOK ), ).once() + flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').with_args( + object, + module.borgmatic.borg.pattern.Pattern( + 'linkdest.yaml', source=module.borgmatic.borg.pattern.Pattern_source.HOOK + ), + ).once() module.dump_data_sources( hook_config=None, @@ -46,6 +94,7 @@ def test_dump_data_sources_creates_manifest_file(): def test_dump_data_sources_with_store_config_files_false_does_not_create_manifest_file(): + flexmock(module).should_receive('resolve_config_path_symlinks').and_yield('test.yaml') flexmock(module.os).should_receive('makedirs').never() flexmock(module.json).should_receive('dump').never() flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').never() @@ -62,6 +111,7 @@ def test_dump_data_sources_with_store_config_files_false_does_not_create_manifes def test_dump_data_sources_with_dry_run_does_not_create_manifest_file(): + flexmock(module).should_receive('resolve_config_path_symlinks').and_yield('test.yaml') flexmock(module.os).should_receive('makedirs').never() flexmock(module.json).should_receive('dump').never() flexmock(module.borgmatic.hooks.data_source.config).should_receive('inject_pattern').never() From 3473f034aec0bac99e687fd7e0b3da99918ace4c Mon Sep 17 00:00:00 2001 From: Dan Helfman Date: Sun, 22 Feb 2026 21:25:27 -0800 Subject: [PATCH 7/7] Fix the "spot" check to skip hard links, as Borg doesn't produces hashes for them (#1236). --- NEWS | 1 + borgmatic/actions/check.py | 39 +++++++------ tests/unit/actions/test_check.py | 95 +++++++++++++++++++++++--------- 3 files changed, 93 insertions(+), 42 deletions(-) diff --git a/NEWS b/NEWS index 285a9093..97499bde 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,7 @@ handy for fetching borgmatic configuration from external scripts. See the documentation for more information: https://torsion.org/borgmatic/reference/command-line/actions/config-show/ + * #1236: Fix the "spot" check to skip hard links, as Borg doesn't produces hashes for them. * #1269: Fix the ZFS hook to support datasets with a "canmount" property of "noauto". * #1270: Follow symlinks when backing up borgmatic configuration files to support the "bootstrap" action. diff --git a/borgmatic/actions/check.py b/borgmatic/actions/check.py index ee0a918e..ce0d894f 100644 --- a/borgmatic/actions/check.py +++ b/borgmatic/actions/check.py @@ -525,6 +525,7 @@ def compare_spot_check_hashes( source_sample_paths_subset = tuple( itertools.islice(source_sample_paths_iterator, SAMPLE_PATHS_SUBSET_COUNT), ) + if not source_sample_paths_subset: break @@ -597,23 +598,27 @@ def compare_spot_check_hashes( source_hashes[hash_path] = '' # Get the hash for each file in the archive. - archive_hashes.update( - **{ - entry['path']: entry['xxh64'] - for entry in borgmatic.borg.list.capture_archive_listing( - repository['path'], - archive, - config, - local_borg_version, - global_arguments, - list_paths=source_sample_paths_subset, - path_format='{xxh64}{path}', - local_path=local_path, - remote_path=remote_path, - ) - if entry - }, - ) + for entry in borgmatic.borg.list.capture_archive_listing( + repository['path'], + archive, + config, + local_borg_version, + global_arguments, + list_paths=source_sample_paths_subset, + path_format='{xxh64}{path}{linktarget}', + local_path=local_path, + remote_path=remote_path, + ): + if not entry: + continue + + # Borg can't get hashes of stored hard links. So if this is a hard link path (and not + # deemed as the "original" by Borg), then skip hashing of it. + if entry['linktarget']: + source_hashes.pop(os.path.join('/', entry['path']), None) + continue + + archive_hashes[entry['path']] = entry['xxh64'] # Compare the source hashes with the archive hashes to see how many match. failing_paths = [] diff --git a/tests/unit/actions/test_check.py b/tests/unit/actions/test_check.py index 8aa6552f..48b98ba0 100644 --- a/tests/unit/actions/test_check.py +++ b/tests/unit/actions/test_check.py @@ -1061,8 +1061,9 @@ def test_compare_spot_check_hashes_returns_paths_having_failing_hashes(): 'hash2 /bar', ) flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield( - {'xxh64': 'hash1', 'path': 'foo'}, - {'xxh64': 'nothash2', 'path': 'bar'}, + {'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''}, + {}, + {'xxh64': 'nothash2', 'path': 'bar', 'linktarget': ''}, ) assert module.compare_spot_check_hashes( @@ -1104,8 +1105,8 @@ def test_compare_spot_check_hashes_handles_weird_backslashed_hashes_from_xxh64su '\\hash2 /bar', ) flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield( - {'xxh64': 'hash1', 'path': 'foo'}, - {'xxh64': 'nothash2', 'path': 'bar'}, + {'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''}, + {'xxh64': 'nothash2', 'path': 'bar', 'linktarget': ''}, ) assert module.compare_spot_check_hashes( @@ -1147,8 +1148,8 @@ def test_compare_spot_check_hashes_handles_incorrect_path_names_from_xxh64sum(): 'hash2 /bar/wrong/path', ) flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield( - {'xxh64': 'hash1', 'path': 'foo'}, - {'xxh64': 'nothash2', 'path': 'bar'}, + {'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''}, + {'xxh64': 'nothash2', 'path': 'bar', 'linktarget': ''}, ) assert module.compare_spot_check_hashes( @@ -1200,8 +1201,8 @@ def test_compare_spot_check_hashes_with_xxh64sum_failure_falls_back_to_individua ).once() flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield( - {'xxh64': 'hash1', 'path': 'foo'}, - {'xxh64': 'hash2', 'path': 'bar'}, + {'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''}, + {'xxh64': 'hash2', 'path': 'bar', 'linktarget': ''}, ) assert module.compare_spot_check_hashes( @@ -1243,8 +1244,8 @@ def test_compare_spot_check_hashes_returns_relative_paths_having_failing_hashes( 'hash2 bar', ) flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield( - {'xxh64': 'hash1', 'path': 'foo'}, - {'xxh64': 'nothash2', 'path': 'bar'}, + {'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''}, + {'xxh64': 'nothash2', 'path': 'bar', 'linktarget': ''}, ) assert module.compare_spot_check_hashes( @@ -1286,8 +1287,8 @@ def test_compare_spot_check_hashes_handles_data_sample_percentage_above_100(): 'hash2 /bar', ) flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield( - {'xxh64': 'nothash1', 'path': 'foo'}, - {'xxh64': 'nothash2', 'path': 'bar'}, + {'xxh64': 'nothash1', 'path': 'foo', 'linktarget': ''}, + {'xxh64': 'nothash2', 'path': 'bar', 'linktarget': ''}, ) assert module.compare_spot_check_hashes( @@ -1329,8 +1330,8 @@ def test_compare_spot_check_hashes_uses_xxh64sum_command_option(): working_directory=None, ).and_yield('hash1 /foo', 'hash2 /bar') flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield( - {'xxh64': 'hash1', 'path': 'foo'}, - {'xxh64': 'nothash2', 'path': 'bar'}, + {'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''}, + {'xxh64': 'nothash2', 'path': 'bar', 'linktarget': ''}, ) assert module.compare_spot_check_hashes( @@ -1369,7 +1370,7 @@ def test_compare_spot_check_hashes_considers_path_missing_from_archive_as_not_ma 'hash2 /bar', ) flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield( - {'xxh64': 'hash1', 'path': 'foo'} + {'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''} ) assert module.compare_spot_check_hashes( @@ -1391,6 +1392,50 @@ def test_compare_spot_check_hashes_considers_path_missing_from_archive_as_not_ma ) == ('/bar',) +def test_compare_spot_check_hashes_skips_hardlink_path_in_archive(): + flexmock(module.random).should_receive('SystemRandom').and_return( + flexmock(sample=lambda population, count: population[:count]), + ) + flexmock(module.borgmatic.config.paths).should_receive('get_working_directory').and_return( + None, + ) + flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.os.path).should_receive('islink').and_return(False) + flexmock(module.borgmatic.execute).should_receive( + 'execute_command_and_capture_output', + ).with_args(('xxh64sum', '/foo', '/bar', '/link'), working_directory=None).and_yield( + 'hash1 /foo', + 'hash2 /bar', + 'hash1 /link', + ) + flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield( + {'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''}, + {'xxh64': 'hash2', 'path': 'bar', 'linktarget': ''}, + {'xxh64': '', 'path': 'link', 'linktarget': 'foo'}, + ) + + assert ( + module.compare_spot_check_hashes( + repository={'path': 'repo'}, + archive='archive', + config={ + 'checks': [ + { + 'name': 'spot', + 'data_sample_percentage': 100, + }, + ], + }, + local_borg_version=flexmock(), + global_arguments=flexmock(), + local_path=flexmock(), + remote_path=flexmock(), + source_paths=('/foo', '/bar', '/link'), + ) + == () + ) + + def test_compare_spot_check_hashes_considers_symlink_path_as_not_matching(): flexmock(module.random).should_receive('SystemRandom').and_return( flexmock(sample=lambda population, count: population[:count]), @@ -1405,8 +1450,8 @@ def test_compare_spot_check_hashes_considers_symlink_path_as_not_matching(): 'execute_command_and_capture_output', ).with_args(('xxh64sum', '/foo'), working_directory=None).and_yield('hash1 /foo') flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield( - {'xxh64': 'hash1', 'path': 'foo'}, - {'xxh64': 'hash2', 'path': 'bar'}, + {'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''}, + {'xxh64': 'hash2', 'path': 'bar', 'linktarget': ''}, ) assert module.compare_spot_check_hashes( @@ -1442,8 +1487,8 @@ def test_compare_spot_check_hashes_considers_non_existent_path_as_not_matching() 'execute_command_and_capture_output', ).with_args(('xxh64sum', '/foo'), working_directory=None).and_yield('hash1 /foo') flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield( - {'xxh64': 'hash1', 'path': 'foo'}, - {'xxh64': 'hash2', 'path': 'bar'}, + {'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''}, + {'xxh64': 'hash2', 'path': 'bar', 'linktarget': ''}, ) assert module.compare_spot_check_hashes( @@ -1488,11 +1533,11 @@ def test_compare_spot_check_hashes_with_too_many_paths_feeds_them_to_commands_in 'hash4 /quux', ) flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield( - {'xxh64': 'hash1', 'path': 'foo'}, - {'xxh64': 'hash2', 'path': 'bar'}, + {'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''}, + {'xxh64': 'hash2', 'path': 'bar', 'linktarget': ''}, ).and_yield( - {'xxh64': 'hash3', 'path': 'baz'}, - {'xxh64': 'nothash4', 'path': 'quux'}, + {'xxh64': 'hash3', 'path': 'baz', 'linktarget': ''}, + {'xxh64': 'nothash4', 'path': 'quux', 'linktarget': ''}, ) assert module.compare_spot_check_hashes( @@ -1535,8 +1580,8 @@ def test_compare_spot_check_hashes_uses_working_directory_to_access_source_paths 'hash2 bar', ) flexmock(module.borgmatic.borg.list).should_receive('capture_archive_listing').and_yield( - {'xxh64': 'hash1', 'path': 'foo'}, - {'xxh64': 'nothash2', 'path': 'bar'}, + {'xxh64': 'hash1', 'path': 'foo', 'linktarget': ''}, + {'xxh64': 'nothash2', 'path': 'bar', 'linktarget': ''}, ) assert module.compare_spot_check_hashes(