diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index e95ec5f4..b09c5660 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1,5 +1,7 @@ import collections import itertools +import json +import re import sys from argparse import ArgumentParser @@ -282,12 +284,155 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse ) -def make_parsers(): +def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names=None): ''' - Build a global arguments parser, individual action parsers, and a combined parser containing - both. Return them as a tuple. The global parser is useful for parsing just global arguments - while ignoring actions, and the combined parser is handy for displaying help that includes - everything: global flags, a list of actions, etc. + Given an argparse._ArgumentGroup instance, a configuration schema dict, and a sequence of + unparsed argument strings, convert the entire schema into corresponding command-line flags and + add them to the arguments group. + + For instance, given a schema of: + + { + 'type': 'object', + 'properties': { + 'foo': { + 'type': 'object', + 'properties': { + 'bar': {'type': 'integer'} + } + } + } + } + + ... the following flag will be added to the arguments group: + + --foo.bar + + If "foo" is instead an array of objects, it will get added like this + + --foo[0].bar + + And if names are also passed in, they are considered to be the name components of an option + (e.g. "foo" and "bar") and are used to construct a resulting flag. + ''' + if names is None: + names = () + + schema_type = schema.get('type') + + # If this option has multiple types, just use the first one (that isn't "null"). + if isinstance(schema_type, list): + try: + schema_type = next(single_type for single_type in schema_type if single_type != 'null') + except StopIteration: + raise ValueError(f'Unknown type in configuration schema: {schema_type}') + + # If this is an "object" type, recurse for each child option ("property"). + if schema_type in {'object', 'array'}: + properties = ( + schema.get('items', {}).get('properties') + if schema_type == 'array' + else schema.get('properties') + ) + + if properties: + for name, child in properties.items(): + add_arguments_from_schema( + arguments_group, + child, + unparsed_arguments, + names + ((name + '[0]',) if child.get('type') == 'array' else (name,)), + ) + + return + + flag_name = '.'.join(names) + description = schema.get('description') + metavar = names[-1].upper() + + if schema_type == 'array': + metavar = metavar.rstrip('S') + + if description: + if schema_type == 'array': + items_schema = schema.get('items', {}) + + description += ' Can specify flag multiple times.' + + if '[0]' in flag_name: + description += ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).' + + description = description.replace('%', '%%') + + try: + argument_type = {'string': str, 'integer': int, 'boolean': bool, 'array': str}[schema_type] + except KeyError: + raise ValueError(f'Unknown type in configuration schema: {schema_type}') + + arguments_group.add_argument( + f"--{flag_name.replace('_', '-')}", + type=argument_type, + metavar=metavar, + action='append' if schema_type == 'array' else None, + help=description, + ) + + # We want to support flags that can have arbitrary indices like: + # + # --foo.bar[1].baz + # + # But argparse doesn't support that natively because the index can be an arbitrary number. We + # won't let that stop us though, will we? So, if the current flag name has an array component in + # it (e.g. a name with "[0]"), then make a pattern that would match the flag name regardless of + # the number that's in it. The idea is that we want to look for unparsed arguments that appear + # like the flag name, but instead of "[0]" they have, say, "[1]" or "[123]". + # + # Next, we check each unparsed argument against that pattern. If one of them matches, add an + # argument flag for it to the argument parser group. Example: + # + # Let's say flag_name is: + # + # --foo.bar[0].baz + # + # ... then the regular expression pattern will be: + # + # ^--foo\.bar\[\d+\]\.baz + # + # ... and, if that matches an unparsed argument of: + # + # --foo.bar[1].baz + # + # ... then an argument flag will get added equal to that unparsed argument. And the unparsed + # argument will match it when parsing is performed! In this manner, we're using the actual user + # CLI input to inform what exact flags we support! + if '[0]' not in flag_name or '--help' in unparsed_arguments: + return + + pattern = re.compile(f'^--{flag_name.replace("[0]", r"\[\d+\]").replace(".", r"\.")}$') + existing_flags = set( + itertools.chain( + *(group_action.option_strings for group_action in arguments_group._group_actions) + ) + ) + + for unparsed in unparsed_arguments: + unparsed_flag_name = unparsed.split('=', 1)[0] + + if pattern.match(unparsed_flag_name) and unparsed_flag_name not in existing_flags: + arguments_group.add_argument( + unparsed_flag_name, + type=argument_type, + metavar=metavar, + help=description, + ) + + +def make_parsers(schema, unparsed_arguments): + ''' + Given a configuration schema dict, build a global arguments parser, individual action parsers, + and a combined parser containing both. Return them as a tuple. The global parser is useful for + parsing just global arguments while ignoring actions, and the combined parser is handy for + displaying help that includes everything: global flags, a list of actions, etc. ''' config_paths = collect.get_default_config_paths(expand_home=True) unexpanded_config_paths = collect.get_default_config_paths(expand_home=False) @@ -388,6 +533,7 @@ def make_parsers(): action='store_true', help='Display installed version number of borgmatic and exit', ) + add_arguments_from_schema(global_group, schema, unparsed_arguments) global_plus_action_parser = ArgumentParser( description=''' @@ -1523,15 +1669,18 @@ def make_parsers(): return global_parser, action_parsers, global_plus_action_parser -def parse_arguments(*unparsed_arguments): +def parse_arguments(schema, *unparsed_arguments): ''' - Given command-line arguments with which this script was invoked, parse the arguments and return - them as a dict mapping from action name (or "global") to an argparse.Namespace instance. + Given a configuration schema dict and the command-line arguments with which this script was + invoked, parse the arguments and return them as a dict mapping from action name (or "global") to + an argparse.Namespace instance. Raise ValueError if the arguments cannot be parsed. Raise SystemExit with an error code of 0 if "--help" was requested. ''' - global_parser, action_parsers, global_plus_action_parser = make_parsers() + global_parser, action_parsers, global_plus_action_parser = make_parsers( + schema, unparsed_arguments + ) arguments, remaining_action_arguments = parse_arguments_for_actions( unparsed_arguments, action_parsers.choices, global_parser ) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 5691df71..9cffbdf8 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -8,6 +8,8 @@ import time from queue import Queue from subprocess import CalledProcessError +import ruamel.yaml + import borgmatic.actions.borg import borgmatic.actions.break_lock import borgmatic.actions.change_passphrase @@ -33,6 +35,7 @@ import borgmatic.actions.restore import borgmatic.actions.transfer import borgmatic.commands.completion.bash import borgmatic.commands.completion.fish +import borgmatic.config.load from borgmatic.borg import umount as borg_umount from borgmatic.borg import version as borg_version from borgmatic.commands.arguments import parse_arguments @@ -570,14 +573,14 @@ def run_actions( ) -def load_configurations(config_filenames, overrides=None, resolve_env=True): +def load_configurations(config_filenames, global_arguments, overrides=None, resolve_env=True): ''' - Given a sequence of configuration filenames, a sequence of configuration file override strings - in the form of "option.suboption=value", and whether to resolve environment variables, load and - validate each configuration file. Return the results as a tuple of: dict of configuration - filename to corresponding parsed configuration, a sequence of paths for all loaded configuration - files (including includes), and a sequence of logging.LogRecord instances containing any parse - errors. + Given a sequence of configuration filenames, global arguments as an argparse.Namespace, a + sequence of configuration file override strings in the form of "option.suboption=value", and + whether to resolve environment variables, load and validate each configuration file. Return the + results as a tuple of: dict of configuration filename to corresponding parsed configuration, a + sequence of paths for all loaded configuration files (including includes), and a sequence of + logging.LogRecord instances containing any parse errors. Log records are returned here instead of being logged directly because logging isn't yet initialized at this point! (Although with the Delayed_logging_handler now in place, maybe this @@ -605,6 +608,7 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True): configs[config_filename], paths, parse_logs = validate.parse_configuration( config_filename, validate.schema_filename(), + global_arguments, overrides, resolve_env, ) @@ -928,9 +932,17 @@ def exit_with_help_link(): # pragma: no cover def main(extra_summary_logs=[]): # pragma: no cover configure_signals() configure_delayed_logging() + schema_filename = validate.schema_filename() try: - arguments = parse_arguments(*sys.argv[1:]) + schema = borgmatic.config.load.load_configuration(schema_filename) + except (ruamel.yaml.error.YAMLError, RecursionError) as error: + configure_logging(logging.CRITICAL) + logger.critical(error) + exit_with_help_link() + + try: + arguments = parse_arguments(schema, *sys.argv[1:]) except ValueError as error: configure_logging(logging.CRITICAL) logger.critical(error) @@ -953,10 +965,10 @@ def main(extra_summary_logs=[]): # pragma: no cover print(borgmatic.commands.completion.fish.fish_completion()) sys.exit(0) - validate = bool('validate' in arguments) config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths)) configs, config_paths, parse_logs = load_configurations( config_filenames, + global_arguments, global_arguments.overrides, resolve_env=global_arguments.resolve_env and not validate, ) diff --git a/borgmatic/config/arguments.py b/borgmatic/config/arguments.py new file mode 100644 index 00000000..96a28b4d --- /dev/null +++ b/borgmatic/config/arguments.py @@ -0,0 +1,137 @@ +import io +import re + +import ruamel.yaml + + +LIST_INDEX_KEY_PATTERN = re.compile(r'^(?P[a-zA-z-]+)\[(?P\d+)\]$') + + +def set_values(config, keys, value): + ''' + Given a configuration dict, a sequence of parsed key strings, and a string value, descend into + the configuration hierarchy based on the keys to set the value into the right place. + ''' + if not keys: + return + + first_key = keys[0] + + # Support "name[0]"-style list index syntax. + match = LIST_INDEX_KEY_PATTERN.match(first_key) + + if match: + list_key = match.group('list_name') + list_index = int(match.group('index')) + + if len(keys) == 1: + config[list_key][list_index] = value + + return + + if list_key not in config: + config[list_key] = [] + + try: + set_values(config[list_key][list_index], keys[1:], value) + except IndexError: + raise ValueError(f'The list index {first_key} is out of range') + + return + + if len(keys) == 1: + config[first_key] = value + return + + if first_key not in config: + config[first_key] = {} + + set_values(config[first_key], keys[1:], value) + + +def type_for_option(schema, option_keys): + ''' + Given a configuration schema dict and a sequence of keys identifying a potentially nested + option, e.g. ('extra_borg_options', 'create'), return the schema type of that option as a + string. + + Return None if the option or its type cannot be found in the schema. + ''' + option_schema = schema + + for key in option_keys: + # Support "name[0]"-style list index syntax. + match = LIST_INDEX_KEY_PATTERN.match(key) + + try: + if match: + option_schema = option_schema['properties'][match.group('list_name')]['items'] + else: + option_schema = option_schema['properties'][key] + except KeyError: + return None + + try: + return option_schema['type'] + except KeyError: + return None + + +def prepare_arguments_for_config(global_arguments, schema): + ''' + Given global arguments as an argparse.Namespace and a configuration schema dict, parse each + argument that corresponds to an option in the schema and return a sequence of tuples (keys, + values) for that option, where keys is a sequence of strings. For instance, given the following + arguments: + + argparse.Namespace(**{'my_option.sub_option': 'value1', 'other_option': 'value2'}) + + ... return this: + + ( + (('my_option', 'sub_option'), 'value1'), + (('other_option'), 'value2'), + ) + + Raise ValueError if an override can't be parsed. + ''' + prepared_values = [] + + for argument_name, value in global_arguments.__dict__.items(): + try: + if value is None: + continue + + keys = tuple(argument_name.split('.')) + option_type = type_for_option(schema, keys) + + # The argument doesn't correspond to any option in the schema, so ignore it. It's + # probably a flag that borgmatic has on the command-line but not in configuration. + if option_type is None: + continue + + prepared_values.append( + ( + keys, + value, + ) + ) + except ruamel.yaml.error.YAMLError as error: + raise ValueError(f"Invalid override '{raw_override}': {error.problem}") + + return tuple(prepared_values) + + +def apply_arguments_to_config(config, schema, global_arguments): + ''' + Given a configuration dict, a corresponding configuration schema dict, and global arguments as + an argparse.Namespace, set those given argument values into their corresponding configuration + options in the configuration dict. + + This supports argument flags of the from "--foo.bar.baz" where each dotted component is a nested + configuration object. Additionally, flags like "--foo.bar[0].baz" are supported to update a list + element in the configuration. + ''' + + for keys, value in prepare_arguments_for_config(global_arguments, schema): + set_values(config, keys, value) diff --git a/borgmatic/config/override.py b/borgmatic/config/override.py index 8a60cb50..0893f669 100644 --- a/borgmatic/config/override.py +++ b/borgmatic/config/override.py @@ -1,8 +1,12 @@ import io +import logging import ruamel.yaml +logger = logging.getLogger(__name__) + + def set_values(config, keys, value): ''' Given a hierarchy of configuration dicts, a sequence of parsed key strings, and a string value, @@ -134,6 +138,9 @@ def apply_overrides(config, schema, raw_overrides): ''' overrides = parse_overrides(raw_overrides, schema) + if overrides: + logger.warning("The --override flag is deprecated and will be removed from a future release. Instead, use a command-line flag corresponding to the configuration option you'd like to set.") + for keys, value in overrides: set_values(config, keys, value) set_values(config, strip_section_names(keys), value) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 13ab0686..3c1bb20f 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -36,9 +36,14 @@ properties: properties: path: type: string + description: The local path or Borg URL of the repository. example: ssh://user@backupserver/./{fqdn} label: type: string + description: | + An optional label for the repository, used in logging + and to make selecting the repository easier on the + command-line. example: backupserver description: | A required list of local or remote repositories with paths and diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index c80cee4d..35c8e809 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -4,7 +4,7 @@ import os import jsonschema import ruamel.yaml -import borgmatic.config +import borgmatic.config.arguments from borgmatic.config import constants, environment, load, normalize, override @@ -84,13 +84,15 @@ def apply_logical_validation(config_filename, parsed_configuration): ) -def parse_configuration(config_filename, schema_filename, overrides=None, resolve_env=True): +def parse_configuration(config_filename, schema_filename, global_arguments, overrides=None, resolve_env=True): ''' Given the path to a config filename in YAML format, the path to a schema filename in a YAML - rendition of JSON Schema format, a sequence of configuration file override strings in the form - of "option.suboption=value", and whether to resolve environment variables, return the parsed - configuration as a data structure of nested dicts and lists corresponding to the schema. Example - return value: + rendition of JSON Schema format, global arguments as an argparse.Namespace, a sequence of + configuration file override strings in the form of "option.suboption=value", and whether to + resolve environment variables, return the parsed configuration as a data structure of nested + dicts and lists corresponding to the schema. Example return value. + + Example return value: { 'source_directories': ['/home', '/etc'], @@ -113,6 +115,7 @@ def parse_configuration(config_filename, schema_filename, overrides=None, resolv except (ruamel.yaml.error.YAMLError, RecursionError) as error: raise Validation_error(config_filename, (str(error),)) + borgmatic.config.arguments.apply_arguments_to_config(config, schema, global_arguments) override.apply_overrides(config, schema, overrides) constants.apply_constants(config, config.get('constants') if config else {})