diff --git a/NEWS b/NEWS index 51724748..1d854410 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,9 @@ +2.1.3.dev0 + * #1218: Add a "config show" action to display computed borgmatic configuration as YAML or JSON, + handy for fetching borgmatic configuration from external scripts. See the documentation for more + information: + https://torsion.org/borgmatic/reference/command-line/actions/config-show/ + 2.1.2 * #1231: If a source file is deleted during a "spot" check, consider the file as non-matching and move on instead of immediately failing the entire check. diff --git a/borgmatic/actions/config/show.py b/borgmatic/actions/config/show.py new file mode 100644 index 00000000..bfa05b0d --- /dev/null +++ b/borgmatic/actions/config/show.py @@ -0,0 +1,44 @@ +import json +import logging +import sys + +import borgmatic.config.generate +import borgmatic.logger + +logger = logging.getLogger(__name__) + + +def run_show(show_arguments, configs): + ''' + Given the show arguments as an argparse.Namespace instance and a dict of configuration filename + to corresponding parsed configuration, run the "show" action. That consists of rendering and + logging the computed configuration as YAML, separating the configuration for each file with + "---". + + If show_arguments.option is set, limit the results to the value of that single option. If + show_arguments.json is True, render the results as JSON with one array element per configuration + file. + ''' + borgmatic.logger.add_custom_log_levels() + + if show_arguments.json: + sys.stdout.write( + json.dumps( + [ + config.get(show_arguments.option) if show_arguments.option else config + for config in configs.values() + ] + ) + ) + + return + + for config in configs.values(): + if len(configs) > 1: + logger.answer('---') + + logger.answer( + borgmatic.config.generate.render_configuration( + config.get(show_arguments.option) if show_arguments.option else config + ).rstrip() + ) diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 58c3a632..73a71556 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1261,6 +1261,29 @@ def make_parsers(schema, unparsed_arguments): # noqa: PLR0915 help='Show this help message and exit', ) + config_show_parser = config_parsers.add_parser( + 'show', + help='Show the computed configuration for each file specified with --config (see borgmatic --help)', + description='Show the computed configuration for each file specified with --config (see borgmatic --help)', + add_help=False, + ) + config_show_group = config_show_parser.add_argument_group('config show arguments') + config_show_group.add_argument( + '--option', + help='Show the value of a single named configuration option instead of the entire configuration', + ) + config_show_group.add_argument( + '--json', + action='store_true', + help='Show the configuration as JSON with one array element per configuration file', + ) + config_show_group.add_argument( + '-h', + '--help', + action='help', + help='Show this help message and exit', + ) + export_tar_parser = action_parsers.add_parser( 'export-tar', aliases=ACTION_ALIASES['export-tar'], diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 2954ac72..7ddaea8c 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -17,6 +17,7 @@ import borgmatic.actions.check import borgmatic.actions.compact import borgmatic.actions.config.bootstrap import borgmatic.actions.config.generate +import borgmatic.actions.config.show import borgmatic.actions.config.validate import borgmatic.actions.create import borgmatic.actions.delete @@ -810,18 +811,18 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par ''' add_custom_log_levels() - if 'bootstrap' in arguments: - try: - # No configuration file is needed for bootstrap. - local_borg_version = borg_version.local_borg_version( - {}, - arguments['bootstrap'].local_path, - ) - except (OSError, CalledProcessError, ValueError) as error: - yield from log_error_records('Error getting local Borg version', error) - return + try: + if 'bootstrap' in arguments: + try: + # No configuration file is needed for bootstrap. + local_borg_version = borg_version.local_borg_version( + {}, + arguments['bootstrap'].local_path, + ) + except (OSError, CalledProcessError, ValueError) as error: + yield from log_error_records('Error getting local Borg version', error) + return - try: borgmatic.actions.config.bootstrap.run_bootstrap( arguments['bootstrap'], arguments['global'], @@ -835,17 +836,10 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par name=logger.name, ), ) - except ( - CalledProcessError, - ValueError, - OSError, - ) as error: - yield from log_error_records(error) - return + return - if 'generate' in arguments: - try: + if 'generate' in arguments: borgmatic.actions.config.generate.run_generate( arguments['generate'], arguments['global'], @@ -858,29 +852,22 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par name=logger.name, ), ) - except ( - CalledProcessError, - ValueError, - OSError, - ) as error: - yield from log_error_records(error) - - return - - if 'validate' in arguments: - if configuration_parse_errors: - yield logging.makeLogRecord( - dict( - levelno=logging.CRITICAL, - levelname='CRITICAL', - msg='Configuration validation failed', - name=logger.name, - ), - ) return - try: + if 'validate' in arguments: + if configuration_parse_errors: + yield logging.makeLogRecord( + dict( + levelno=logging.CRITICAL, + levelname='CRITICAL', + msg='Configuration validation failed', + name=logger.name, + ), + ) + + return + borgmatic.actions.config.validate.run_validate(arguments['validate'], configs) yield logging.makeLogRecord( @@ -891,14 +878,20 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par name=logger.name, ), ) - except ( - CalledProcessError, - ValueError, - OSError, - ) as error: - yield from log_error_records(error) - return + return + + if 'show' in arguments: + borgmatic.actions.config.show.run_show(arguments['show'], configs) + + return + + except ( + CalledProcessError, + ValueError, + OSError, + ) as error: + yield from log_error_records(error) def collect_configuration_run_summary_logs(configs, config_paths, arguments, log_file_path): # noqa: PLR0912 diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index 76d4a167..4d3cda78 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -153,6 +153,9 @@ def transform_optional_configuration(rendered_config, comment_out=True): return '\n'.join(lines) +RUAMEL_YAML_END_OF_DOCUMENT_MARKER = '...\n' + + def render_configuration(config): ''' Given a config data structure of nested OrderedDicts, render the config as YAML and return it. @@ -160,7 +163,15 @@ def render_configuration(config): dumper = ruamel.yaml.YAML(typ='rt') dumper.indent(mapping=INDENT, sequence=INDENT + SEQUENCE_INDENT, offset=INDENT) rendered = io.StringIO() - dumper.dump(config, rendered) + dumper.dump( + config, + rendered, + # Dumping certain values (integers, for instance) causes ruamel.yaml to append an + # end-of-document "..." marker. Strip it. + transform=lambda dumped: dumped[: -len(RUAMEL_YAML_END_OF_DOCUMENT_MARKER)] + if dumped.endswith(RUAMEL_YAML_END_OF_DOCUMENT_MARKER) + else dumped, + ) return rendered.getvalue() diff --git a/docs/Dockerfile b/docs/Dockerfile index 5463b98d..a58c92c0 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -5,7 +5,7 @@ RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib RUN pip install --break-system-packages --no-cache /app && borgmatic config generate && borgmatic config generate --destination /etc/borgmatic --split && chmod +r /etc/borgmatic/*.yaml RUN mkdir /command-line \ && borgmatic --help > /command-line/global.txt \ - && for action in repo-create transfer create prune compact check delete extract config "config bootstrap" "config generate" "config validate" export-tar mount umount repo-delete restore repo-list list repo-info info break-lock "key export" "key import" "key change-passphrase" recreate borg; do \ + && for action in repo-create transfer create prune compact check delete extract config "config bootstrap" "config generate" "config validate" "config show" export-tar mount umount repo-delete restore repo-list list repo-info info break-lock "key export" "key import" "key change-passphrase" recreate borg; do \ borgmatic $action --help > /command-line/${action/ /-}.txt; done RUN /app/docs/fetch-contributors >> /contributors.html diff --git a/docs/how-to/monitor-your-backups.md b/docs/how-to/monitor-your-backups.md index 763162fc..963e07d2 100644 --- a/docs/how-to/monitor-your-backups.md +++ b/docs/how-to/monitor-your-backups.md @@ -64,6 +64,27 @@ suppressed so as not to interfere with the captured JSON. Also note that JSON output only shows up at the console and not in syslog. +### Getting configuration + +New in version 2.1.3 If you want +to consume borgmatic's computed configuration in your scripts, use the [`config +show` +action](https://torsion.org/borgmatic/reference/command-line/actions/config-show/). +Here's an example: + +```bash +borgmatic config show --json +``` + +That outputs borgmatic's entire configuration as JSON with one array element per +configuration file. + +Or you can ask for the value of a particular option: + +```bash +borgmatic config show --option repositories --json +``` + ### Latest backups All borgmatic actions that accept an `--archive` flag allow you to specify an diff --git a/docs/reference/command-line/actions/config-show.md b/docs/reference/command-line/actions/config-show.md new file mode 100644 index 00000000..12ad4552 --- /dev/null +++ b/docs/reference/command-line/actions/config-show.md @@ -0,0 +1,17 @@ +--- +title: config show +eleventyNavigation: + key: config show + parent: 🎬 Actions +--- + +{% include snippet/command-line/sample.md %} + +``` +{% include borgmatic/command-line/config-show.txt %} +``` + + +## Related documentation + + * [Scripting borgmatic](https://torsion.org/borgmatic/how-to/monitor-your-backups/#scripting-borgmatic) diff --git a/docs/reference/configuration/includes.md b/docs/reference/configuration/includes.md index 1b2691fb..2175fe2e 100644 --- a/docs/reference/configuration/includes.md +++ b/docs/reference/configuration/includes.md @@ -317,3 +317,6 @@ a default location. This will output the merged configuration as borgmatic sees it, which can be helpful for understanding how your includes work in practice. + +Also see the [`config show` +action](https://torsion.org/borgmatic/reference/command-line/actions/config-show/). diff --git a/pyproject.toml b/pyproject.toml index 8c0274a2..88a1d506 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "borgmatic" -version = "2.1.2" +version = "2.1.3.dev0" authors = [ { name="Dan Helfman", email="witten@torsion.org" }, ] diff --git a/tests/integration/config/test_generate.py b/tests/integration/config/test_generate.py index 48fbd502..14b03627 100644 --- a/tests/integration/config/test_generate.py +++ b/tests/integration/config/test_generate.py @@ -292,6 +292,12 @@ def test_render_configuration_converts_configuration_to_yaml_string(): assert yaml_string == 'foo: bar\n' +def test_render_configuration_strips_ruamel_yaml_end_of_document_marker(): + yaml_string = module.render_configuration(33) + + assert yaml_string == '33\n' + + def test_write_configuration_does_not_raise(): flexmock(os.path).should_receive('exists').and_return(False) flexmock(os).should_receive('makedirs') diff --git a/tests/unit/actions/config/test_show.py b/tests/unit/actions/config/test_show.py new file mode 100644 index 00000000..2f12c7ea --- /dev/null +++ b/tests/unit/actions/config/test_show.py @@ -0,0 +1,79 @@ +from flexmock import flexmock + +import borgmatic.logger +from borgmatic.actions.config import show as module + + +def test_run_show_with_single_configuration_file_does_not_separate_output(): + log_lines = [] + borgmatic.logger.add_custom_log_levels() + + def fake_logger_answer(message): + log_lines.append(message) + + flexmock(module.logger).should_receive('answer').replace_with(fake_logger_answer) + show_arguments = flexmock(option=None, json=False) + flexmock(module.borgmatic.config.generate).should_receive('render_configuration').and_return( + 'output' + ) + + module.run_show(show_arguments, configs={'test.yaml': {}}) + + assert log_lines == ['output'] + + +def test_run_show_with_multiple_configuration_files_separates_output(): + log_lines = [] + borgmatic.logger.add_custom_log_levels() + + def fake_logger_answer(message): + log_lines.append(message) + + flexmock(module.logger).should_receive('answer').replace_with(fake_logger_answer) + show_arguments = flexmock(option=None, json=False) + flexmock(module.borgmatic.config.generate).should_receive('render_configuration').and_return( + 'output' + ).and_return('other') + + module.run_show(show_arguments, configs={'test.yaml': {}, 'other.yaml': {}}) + + assert log_lines == ['---', 'output', '---', 'other'] + + +def test_run_show_with_option_limits_output(): + log_lines = [] + borgmatic.logger.add_custom_log_levels() + + def fake_logger_answer(message): + log_lines.append(message) + + flexmock(module.logger).should_receive('answer').replace_with(fake_logger_answer) + show_arguments = flexmock(option='foo', json=False) + flexmock(module.borgmatic.config.generate).should_receive('render_configuration').with_args( + 33 + ).and_return('33') + flexmock(module.borgmatic.config.generate).should_receive('render_configuration').with_args( + None + ).and_return('null') + + module.run_show(show_arguments, configs={'test.yaml': {'foo': 33, 'bar': 44}, 'other.yaml': {}}) + + assert log_lines == ['---', '33', '---', 'null'] + + +def test_run_show_with_json_outputs_json(): + flexmock(borgmatic.logger).should_receive('add_custom_log_levels') + show_arguments = flexmock(option=None, json=True) + flexmock(module.sys.stdout).should_receive('write').with_args( + '[{"foo": 33}, {"bar": 44}]' + ).once() + + module.run_show(show_arguments, configs={'test.yaml': {'foo': 33}, 'other.yaml': {'bar': 44}}) + + +def test_run_show_with_json_and_option_limits_json(): + flexmock(borgmatic.logger).should_receive('add_custom_log_levels') + show_arguments = flexmock(option='foo', json=True) + flexmock(module.sys.stdout).should_receive('write').with_args('[33, null]').once() + + module.run_show(show_arguments, configs={'test.yaml': {'foo': 33}, 'other.yaml': {'bar': 44}}) diff --git a/tests/unit/actions/config/test_validate.py b/tests/unit/actions/config/test_validate.py index 862d1bfa..f24db6c4 100644 --- a/tests/unit/actions/config/test_validate.py +++ b/tests/unit/actions/config/test_validate.py @@ -5,13 +5,13 @@ from borgmatic.actions.config import validate as module def test_run_validate_does_not_raise(): validate_arguments = flexmock(show=False) - flexmock(module.borgmatic.config.generate).should_receive('render_configuration') + flexmock(module.borgmatic.config.generate).should_receive('render_configuration').and_return('') module.run_validate(validate_arguments, flexmock()) def test_run_validate_with_show_does_not_raise(): validate_arguments = flexmock(show=True) - flexmock(module.borgmatic.config.generate).should_receive('render_configuration') + flexmock(module.borgmatic.config.generate).should_receive('render_configuration').and_return('') module.run_validate(validate_arguments, {'test.yaml': flexmock(), 'other.yaml': flexmock()}) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 887f1631..7caa1596 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -2058,6 +2058,43 @@ def test_collect_highlander_action_summary_logs_error_on_run_validate_failure(): assert {log.levelno for log in logs} == {logging.CRITICAL} +def test_collect_highlander_action_summary_logs_nothing_additional_for_success_with_show(): + flexmock(module.borgmatic.actions.config.show).should_receive('run_show') + arguments = { + 'show': flexmock(), + 'global': flexmock(), + } + + logs = tuple( + module.collect_highlander_action_summary_logs( + {'test.yaml': {}}, + arguments=arguments, + configuration_parse_errors=False, + ), + ) + assert not logs + + +def test_collect_highlander_action_summary_logs_error_on_run_show_failure(): + flexmock(module.borgmatic.actions.config.show).should_receive('run_show').and_raise( + ValueError, + ) + arguments = { + 'show': flexmock(), + 'global': flexmock(), + } + + logs = tuple( + module.collect_highlander_action_summary_logs( + {'test.yaml': {}}, + arguments=arguments, + configuration_parse_errors=False, + ), + ) + + assert {log.levelno for log in logs} == {logging.CRITICAL} + + def test_collect_configuration_run_summary_logs_info_for_success(): flexmock(module.validate).should_receive('guard_configuration_contains_repository') flexmock(module.command).should_receive('filter_hooks').with_args(