diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 6504257b..555f41de 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1,11 +1,16 @@ import collections +import decimal import itertools +import io import json import re import sys from argparse import ArgumentParser from borgmatic.config import collect +import borgmatic.config.schema + +import ruamel.yaml ACTION_ALIASES = { 'repo-create': ['rcreate', 'init', '-I'], @@ -308,8 +313,9 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names --foo.bar - If "foo" is instead an array of objects, it will get added like this + If "foo" is instead an array of objects, both of the following will get added: + --foo --foo[0].bar And if names are also passed in, they are considered to be the name components of an option @@ -328,12 +334,8 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names raise ValueError(f'Unknown type in configuration schema: {schema_type}') # If this is an "object" type, recurse for each child option ("property"). - if schema_type in {'object', 'array'}: - properties = ( - schema.get('items', {}).get('properties') - if schema_type == 'array' - else schema.get('properties') - ) + if schema_type == 'object': + properties = schema.get('properties') if properties: for name, child in properties.items(): @@ -341,23 +343,37 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names arguments_group, child, unparsed_arguments, - names + ((name + '[0]',) if child.get('type') == 'array' else (name,)), + names + (name,) ) return + # If this is an "array" type, recurse for each child option of its items type. Don't return yet, + # so that a flag also gets added below for the array itself. + if schema_type == 'array': + properties = borgmatic.config.schema.get_properties(schema.get('items', {})) + + if properties: + for name, child in properties.items(): + add_arguments_from_schema( + arguments_group, + child, + unparsed_arguments, + names[:-1] + (f'{names[-1]}[0]',) + (name,) + ) + flag_name = '.'.join(names) description = schema.get('description') metavar = names[-1].upper() - if schema_type == 'array': - metavar = metavar.rstrip('S') - if description: if schema_type == 'array': - items_schema = schema.get('items', {}) + example_buffer = io.StringIO() + yaml = ruamel.yaml.YAML(typ='safe') + yaml.default_flow_style = True + yaml.dump(schema.get('example'), example_buffer) - description += ' Can specify flag multiple times.' + description += f' Example value: "{example_buffer.getvalue().strip()}"' if '[0]' in flag_name: description += ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).' @@ -365,7 +381,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names description = description.replace('%', '%%') try: - argument_type = {'string': str, 'integer': int, 'boolean': bool, 'array': str}[schema_type] + argument_type = {'string': str, 'integer': int, 'number': decimal.Decimal, 'boolean': bool, 'array': str}[schema_type] except KeyError: raise ValueError(f'Unknown type in configuration schema: {schema_type}') @@ -373,7 +389,6 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names f"--{flag_name.replace('_', '-')}", type=argument_type, metavar=metavar, - action='append' if schema_type == 'array' else None, help=description, ) diff --git a/borgmatic/config/arguments.py b/borgmatic/config/arguments.py index ec7fe8ee..dab37b5b 100644 --- a/borgmatic/config/arguments.py +++ b/borgmatic/config/arguments.py @@ -21,7 +21,7 @@ def set_values(config, keys, value): ('mylist[0]', 'foo') This looks for the zeroth element of "mylist" in the given configuration. And within that value, - it looks up "foo" and sets it to the given value. Finally: + it looks up "foo" and sets it to the given value. ''' if not keys: return @@ -89,6 +89,22 @@ def type_for_option(schema, option_keys): return None +def convert_value_type(value, option_type): + ''' + Given a string value and its schema type as a string, determine its logical type (string, + boolean, integer, etc.), and return it converted to that type. + + If the option type is a string, leave the value as a string so that special characters in it + don't get interpreted as YAML during conversion. + + Raise ruamel.yaml.error.YAMLError if there's a parse issue with the YAML. + ''' + if option_type == 'string': + return value + + return ruamel.yaml.YAML(typ='safe').load(io.StringIO(value)) + + def prepare_arguments_for_config(global_arguments, schema): ''' Given global arguments as an argparse.Namespace and a configuration schema dict, parse each @@ -125,11 +141,11 @@ def prepare_arguments_for_config(global_arguments, schema): prepared_values.append( ( keys, - value, + convert_value_type(value, option_type), ) ) except ruamel.yaml.error.YAMLError as error: - raise ValueError(f"Invalid override '{raw_override}': {error.problem}") + raise ValueError(f'Invalid override "{argument_name}": {error.problem}') return tuple(prepared_values) diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index 58fd03a7..e0a646d8 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -1,12 +1,12 @@ import collections import io -import itertools import os import re import ruamel.yaml from borgmatic.config import load, normalize +import borgmatic.config.schema INDENT = 4 SEQUENCE_INDENT = 2 @@ -22,27 +22,6 @@ def insert_newline_before_comment(config, field_name): ) -def get_properties(schema): - ''' - Given a schema dict, return its properties. But if it's got sub-schemas with multiple different - potential properties, returned their merged properties instead (interleaved so the first - properties of each sub-schema come first). The idea is that the user should see all possible - options even if they're not all possible together. - ''' - if 'oneOf' in schema: - return dict( - item - for item in itertools.chain( - *itertools.zip_longest( - *[sub_schema['properties'].items() for sub_schema in schema['oneOf']] - ) - ) - if item is not None - ) - - return schema['properties'] - - def schema_to_sample_configuration(schema, source_config=None, level=0, parent_is_sequence=False): ''' Given a loaded configuration schema and a source configuration, generate and return sample @@ -78,7 +57,7 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i sub_schema, (source_config or {}).get(field_name, {}), level + 1 ), ) - for field_name, sub_schema in get_properties(schema).items() + for field_name, sub_schema in borgmatic.config.schema.get_properties(schema).items() ] ) indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0) @@ -189,7 +168,7 @@ def add_comments_to_configuration_sequence(config, schema, indent=0): return for field_name in config[0].keys(): - field_schema = get_properties(schema['items']).get(field_name, {}) + field_schema = borgmatic.config.schema.get_properties(schema['items']).get(field_name, {}) description = field_schema.get('description') # No description to use? Skip it. @@ -223,7 +202,7 @@ def add_comments_to_configuration_object( if skip_first and index == 0: continue - field_schema = get_properties(schema).get(field_name, {}) + field_schema = borgmatic.config.schema.get_properties(schema).get(field_name, {}) description = field_schema.get('description', '').strip() # If this isn't a default key, add an indicator to the comment flagging it to be commented diff --git a/borgmatic/config/normalize.py b/borgmatic/config/normalize.py index 11f21ce0..f4199e6b 100644 --- a/borgmatic/config/normalize.py +++ b/borgmatic/config/normalize.py @@ -326,7 +326,11 @@ def normalize(config_filename, config): config['repositories'] = [] for repository_dict in repositories: - repository_path = repository_dict['path'] + repository_path = repository_dict.get('path') + + if repository_path is None: + continue + if '~' in repository_path: logs.append( logging.makeLogRecord( diff --git a/borgmatic/config/schema.py b/borgmatic/config/schema.py new file mode 100644 index 00000000..e46d27cc --- /dev/null +++ b/borgmatic/config/schema.py @@ -0,0 +1,22 @@ +import itertools + + +def get_properties(schema): + ''' + Given a schema dict, return its properties. But if it's got sub-schemas with multiple different + potential properties, returned their merged properties instead (interleaved so the first + properties of each sub-schema come first). The idea is that the user should see all possible + options even if they're not all possible together. + ''' + if 'oneOf' in schema: + return dict( + item + for item in itertools.chain( + *itertools.zip_longest( + *[sub_schema['properties'].items() for sub_schema in schema['oneOf']] + ) + ) + if item is not None + ) + + return schema.get('properties', {}) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 3c1bb20f..2f892337 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -53,8 +53,7 @@ properties: output of "borg help placeholders" for details. See ssh_command for SSH options like identity file or port. If systemd service is used, then add local repository paths in the systemd service file to the - ReadWritePaths list. Prior to borgmatic 1.7.10, repositories was a - list of plain path strings. + ReadWritePaths list. example: - path: ssh://user@backupserver/./sourcehostname.borg label: backupserver @@ -738,6 +737,10 @@ properties: List of one or more consistency checks to run on a periodic basis (if "frequency" is set) or every time borgmatic runs checks (if "frequency" is omitted). + example: + - name: archives + frequency: 2 weeks + - name: repository check_repositories: type: array items: @@ -1115,6 +1118,10 @@ properties: List of one or more command hooks to execute, triggered at particular points during borgmatic's execution. For each command hook, specify one of "before" or "after", not both. + example: + - before: action + when: [create] + run: [echo Backing up.] bootstrap: type: object properties: @@ -1329,6 +1336,9 @@ properties: https://www.postgresql.org/docs/current/app-pgdump.html and https://www.postgresql.org/docs/current/libpq-ssl.html for details. + example: + - name: users + hostname: database.example.org mariadb_databases: type: array items: @@ -1473,6 +1483,9 @@ properties: added to your source directories at runtime and streamed directly to Borg. Requires mariadb-dump/mariadb commands. See https://mariadb.com/kb/en/library/mysqldump/ for details. + example: + - name: users + hostname: database.example.org mysql_databases: type: array items: @@ -1618,6 +1631,9 @@ properties: to Borg. Requires mysqldump/mysql commands. See https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html for details. + example: + - name: users + hostname: database.example.org sqlite_databases: type: array items: @@ -1647,6 +1663,15 @@ properties: Path to the SQLite database file to restore to. Defaults to the "path" option. example: /var/lib/sqlite/users.db + description: | + List of one or more SQLite databases to dump before creating a + backup, run once per configuration file. The database dumps are + added to your source directories at runtime and streamed directly to + Borg. Requires the sqlite3 command. See https://sqlite.org/cli.html + for details. + example: + - name: users + path: /var/lib/db.sqlite mongodb_databases: type: array items: @@ -1749,6 +1774,9 @@ properties: to Borg. Requires mongodump/mongorestore commands. See https://docs.mongodb.com/database-tools/mongodump/ and https://docs.mongodb.com/database-tools/mongorestore/ for details. + example: + - name: users + hostname: database.example.org ntfy: type: object required: ['topic'] @@ -2231,9 +2259,13 @@ properties: properties: url: type: string + description: URL of this Apprise service. example: "gotify://hostname/token" label: type: string + description: | + Label used in borgmatic logs for this Apprise + service. example: gotify description: | A list of Apprise services to publish to with URLs and diff --git a/tests/unit/config/test_generate.py b/tests/unit/config/test_generate.py index 27dcd01f..abb520fb 100644 --- a/tests/unit/config/test_generate.py +++ b/tests/unit/config/test_generate.py @@ -4,82 +4,6 @@ from flexmock import flexmock from borgmatic.config import generate as module -def test_get_properties_with_simple_object(): - schema = { - 'type': 'object', - 'properties': dict( - [ - ('field1', {'example': 'Example'}), - ] - ), - } - - assert module.get_properties(schema) == schema['properties'] - - -def test_get_properties_merges_oneof_list_properties(): - schema = { - 'type': 'object', - 'oneOf': [ - { - 'properties': dict( - [ - ('field1', {'example': 'Example 1'}), - ('field2', {'example': 'Example 2'}), - ] - ), - }, - { - 'properties': dict( - [ - ('field2', {'example': 'Example 2'}), - ('field3', {'example': 'Example 3'}), - ] - ), - }, - ], - } - - assert module.get_properties(schema) == dict( - schema['oneOf'][0]['properties'], **schema['oneOf'][1]['properties'] - ) - - -def test_get_properties_interleaves_oneof_list_properties(): - schema = { - 'type': 'object', - 'oneOf': [ - { - 'properties': dict( - [ - ('field1', {'example': 'Example 1'}), - ('field2', {'example': 'Example 2'}), - ('field3', {'example': 'Example 3'}), - ] - ), - }, - { - 'properties': dict( - [ - ('field4', {'example': 'Example 4'}), - ('field5', {'example': 'Example 5'}), - ] - ), - }, - ], - } - - assert module.get_properties(schema) == dict( - [ - ('field1', {'example': 'Example 1'}), - ('field4', {'example': 'Example 4'}), - ('field2', {'example': 'Example 2'}), - ('field5', {'example': 'Example 5'}), - ('field3', {'example': 'Example 3'}), - ] - ) - - def test_schema_to_sample_configuration_generates_config_map_with_examples(): schema = { 'type': 'object', diff --git a/tests/unit/config/test_schema.py b/tests/unit/config/test_schema.py new file mode 100644 index 00000000..e10992b6 --- /dev/null +++ b/tests/unit/config/test_schema.py @@ -0,0 +1,80 @@ +from borgmatic.config import schema as module + + +def test_get_properties_with_simple_object(): + schema = { + 'type': 'object', + 'properties': dict( + [ + ('field1', {'example': 'Example'}), + ] + ), + } + + assert module.get_properties(schema) == schema['properties'] + + +def test_get_properties_merges_oneof_list_properties(): + schema = { + 'type': 'object', + 'oneOf': [ + { + 'properties': dict( + [ + ('field1', {'example': 'Example 1'}), + ('field2', {'example': 'Example 2'}), + ] + ), + }, + { + 'properties': dict( + [ + ('field2', {'example': 'Example 2'}), + ('field3', {'example': 'Example 3'}), + ] + ), + }, + ], + } + + assert module.get_properties(schema) == dict( + schema['oneOf'][0]['properties'], **schema['oneOf'][1]['properties'] + ) + + +def test_get_properties_interleaves_oneof_list_properties(): + schema = { + 'type': 'object', + 'oneOf': [ + { + 'properties': dict( + [ + ('field1', {'example': 'Example 1'}), + ('field2', {'example': 'Example 2'}), + ('field3', {'example': 'Example 3'}), + ] + ), + }, + { + 'properties': dict( + [ + ('field4', {'example': 'Example 4'}), + ('field5', {'example': 'Example 5'}), + ] + ), + }, + ], + } + + assert module.get_properties(schema) == dict( + [ + ('field1', {'example': 'Example 1'}), + ('field4', {'example': 'Example 4'}), + ('field2', {'example': 'Example 2'}), + ('field5', {'example': 'Example 5'}), + ('field3', {'example': 'Example 3'}), + ] + ) + + +