peter/borgmatic
1
0
Fork 0
mirror of https://github.com/borgmatic-collective/borgmatic.git synced 2026-07-29 21:03:02 +02:00
Code Issues Packages Projects Releases Wiki Activity

Support setting whole lists and dicts from the command-line (#303).

Browse Source
This commit is contained in:
Dan Helfman
2025-03-17 09:57:25 -07:00
parent 05900c188f
commit eca78fbc2c
8 changed files with 194 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
borgmatic/commands/arguments.py
+30 -15
View File
@@ -1,11 +1,16 @@
import collections
import decimal
import itertools
import io
import json
import re
import sys
from argparse import ArgumentParser
from borgmatic.config import collect
import borgmatic.config.schema
import ruamel.yaml
ACTION_ALIASES = {
'repo-create': ['rcreate', 'init', '-I'],
@@ -308,8 +313,9 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names
--foo.bar
If "foo" is instead an array of objects, it will get added like this
If "foo" is instead an array of objects, both of the following will get added:
--foo
--foo[0].bar
And if names are also passed in, they are considered to be the name components of an option
@@ -328,12 +334,8 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names
raise ValueError(f'Unknown type in configuration schema: {schema_type}')
# If this is an "object" type, recurse for each child option ("property").
if schema_type in {'object', 'array'}:
properties = (
schema.get('items', {}).get('properties')
if schema_type == 'array'
else schema.get('properties')
)
if schema_type == 'object':
properties = schema.get('properties')
if properties:
for name, child in properties.items():
@@ -341,23 +343,37 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names
arguments_group,
child,
unparsed_arguments,
names + ((name + '[0]',) if child.get('type') == 'array' else (name,)),
names + (name,)
)
return
# If this is an "array" type, recurse for each child option of its items type. Don't return yet,
# so that a flag also gets added below for the array itself.
if schema_type == 'array':
properties = borgmatic.config.schema.get_properties(schema.get('items', {}))
if properties:
for name, child in properties.items():
add_arguments_from_schema(
arguments_group,
child,
unparsed_arguments,
names[:-1] + (f'{names[-1]}[0]',) + (name,)
)
flag_name = '.'.join(names)
description = schema.get('description')
metavar = names[-1].upper()
if schema_type == 'array':
metavar = metavar.rstrip('S')
if description:
if schema_type == 'array':
items_schema = schema.get('items', {})
example_buffer = io.StringIO()
yaml = ruamel.yaml.YAML(typ='safe')
yaml.default_flow_style = True
yaml.dump(schema.get('example'), example_buffer)
description += ' Can specify flag multiple times.'
description += f' Example value: "{example_buffer.getvalue().strip()}"'
if '[0]' in flag_name:
description += ' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).'
@@ -365,7 +381,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names
description = description.replace('%', '%%')
try:
argument_type = {'string': str, 'integer': int, 'boolean': bool, 'array': str}[schema_type]
argument_type = {'string': str, 'integer': int, 'number': decimal.Decimal, 'boolean': bool, 'array': str}[schema_type]
except KeyError:
raise ValueError(f'Unknown type in configuration schema: {schema_type}')
@@ -373,7 +389,6 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names
f"--{flag_name.replace('_', '-')}",
type=argument_type,
metavar=metavar,
action='append' if schema_type == 'array' else None,
help=description,
)
borgmatic/config/arguments.py
+19 -3
View File
@@ -21,7 +21,7 @@ def set_values(config, keys, value):
('mylist[0]', 'foo')
This looks for the zeroth element of "mylist" in the given configuration. And within that value,
it looks up "foo" and sets it to the given value. Finally:
it looks up "foo" and sets it to the given value.
'''
if not keys:
return
@@ -89,6 +89,22 @@ def type_for_option(schema, option_keys):
return None
def convert_value_type(value, option_type):
'''
Given a string value and its schema type as a string, determine its logical type (string,
boolean, integer, etc.), and return it converted to that type.
If the option type is a string, leave the value as a string so that special characters in it
don't get interpreted as YAML during conversion.
Raise ruamel.yaml.error.YAMLError if there's a parse issue with the YAML.
'''
if option_type == 'string':
return value
return ruamel.yaml.YAML(typ='safe').load(io.StringIO(value))
def prepare_arguments_for_config(global_arguments, schema):
'''
Given global arguments as an argparse.Namespace and a configuration schema dict, parse each
@@ -125,11 +141,11 @@ def prepare_arguments_for_config(global_arguments, schema):
prepared_values.append(
(
keys,
value,
convert_value_type(value, option_type),
)
)
except ruamel.yaml.error.YAMLError as error:
raise ValueError(f"Invalid override '{raw_override}': {error.problem}")
raise ValueError(f'Invalid override "{argument_name}": {error.problem}')
return tuple(prepared_values)
borgmatic/config/generate.py
+4 -25
View File
@@ -1,12 +1,12 @@
import collections
import io
import itertools
import os
import re
import ruamel.yaml
from borgmatic.config import load, normalize
import borgmatic.config.schema
INDENT = 4
SEQUENCE_INDENT = 2
@@ -22,27 +22,6 @@ def insert_newline_before_comment(config, field_name):
)
def get_properties(schema):
'''
Given a schema dict, return its properties. But if it's got sub-schemas with multiple different
potential properties, returned their merged properties instead (interleaved so the first
properties of each sub-schema come first). The idea is that the user should see all possible
options even if they're not all possible together.
'''
if 'oneOf' in schema:
return dict(
item
for item in itertools.chain(
*itertools.zip_longest(
*[sub_schema['properties'].items() for sub_schema in schema['oneOf']]
)
)
if item is not None
)
return schema['properties']
def schema_to_sample_configuration(schema, source_config=None, level=0, parent_is_sequence=False):
'''
Given a loaded configuration schema and a source configuration, generate and return sample
@@ -78,7 +57,7 @@ def schema_to_sample_configuration(schema, source_config=None, level=0, parent_i
sub_schema, (source_config or {}).get(field_name, {}), level + 1
),
)
for field_name, sub_schema in get_properties(schema).items()
for field_name, sub_schema in borgmatic.config.schema.get_properties(schema).items()
]
)
indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0)
@@ -189,7 +168,7 @@ def add_comments_to_configuration_sequence(config, schema, indent=0):
return
for field_name in config[0].keys():
field_schema = get_properties(schema['items']).get(field_name, {})
field_schema = borgmatic.config.schema.get_properties(schema['items']).get(field_name, {})
description = field_schema.get('description')
# No description to use? Skip it.
@@ -223,7 +202,7 @@ def add_comments_to_configuration_object(
if skip_first and index == 0:
continue
field_schema = get_properties(schema).get(field_name, {})
field_schema = borgmatic.config.schema.get_properties(schema).get(field_name, {})
description = field_schema.get('description', '').strip()
# If this isn't a default key, add an indicator to the comment flagging it to be commented
borgmatic/config/normalize.py
+5 -1
View File
@@ -326,7 +326,11 @@ def normalize(config_filename, config):
config['repositories'] = []
for repository_dict in repositories:
repository_path = repository_dict['path']
repository_path = repository_dict.get('path')
if repository_path is None:
continue
if '~' in repository_path:
logs.append(
logging.makeLogRecord(
borgmatic/config/schema.py
+22
View File
@@ -0,0 +1,22 @@
import itertools
def get_properties(schema):
'''
Given a schema dict, return its properties. But if it's got sub-schemas with multiple different
potential properties, returned their merged properties instead (interleaved so the first
properties of each sub-schema come first). The idea is that the user should see all possible
options even if they're not all possible together.
'''
if 'oneOf' in schema:
return dict(
item
for item in itertools.chain(
*itertools.zip_longest(
*[sub_schema['properties'].items() for sub_schema in schema['oneOf']]
)
)
if item is not None
)
return schema.get('properties', {})
borgmatic/config/schema.yaml
+34 -2
View File
@@ -53,8 +53,7 @@ properties:
output of "borg help placeholders" for details. See ssh_command for
SSH options like identity file or port. If systemd service is used,
then add local repository paths in the systemd service file to the
ReadWritePaths list. Prior to borgmatic 1.7.10, repositories was a
list of plain path strings.
ReadWritePaths list.
example:
- path: ssh://user@backupserver/./sourcehostname.borg
label: backupserver
@@ -738,6 +737,10 @@ properties:
List of one or more consistency checks to run on a periodic basis
(if "frequency" is set) or every time borgmatic runs checks (if
"frequency" is omitted).
example:
- name: archives
frequency: 2 weeks
- name: repository
check_repositories:
type: array
items:
@@ -1115,6 +1118,10 @@ properties:
List of one or more command hooks to execute, triggered at
particular points during borgmatic's execution. For each command
hook, specify one of "before" or "after", not both.
example:
- before: action
when: [create]
run: [echo Backing up.]
bootstrap:
type: object
properties:
@@ -1329,6 +1336,9 @@ properties:
https://www.postgresql.org/docs/current/app-pgdump.html and
https://www.postgresql.org/docs/current/libpq-ssl.html for
details.
example:
- name: users
hostname: database.example.org
mariadb_databases:
type: array
items:
@@ -1473,6 +1483,9 @@ properties:
added to your source directories at runtime and streamed directly
to Borg. Requires mariadb-dump/mariadb commands. See
https://mariadb.com/kb/en/library/mysqldump/ for details.
example:
- name: users
hostname: database.example.org
mysql_databases:
type: array
items:
@@ -1618,6 +1631,9 @@ properties:
to Borg. Requires mysqldump/mysql commands. See
https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html for
details.
example:
- name: users
hostname: database.example.org
sqlite_databases:
type: array
items:
@@ -1647,6 +1663,15 @@ properties:
Path to the SQLite database file to restore to. Defaults
to the "path" option.
example: /var/lib/sqlite/users.db
description: |
List of one or more SQLite databases to dump before creating a
backup, run once per configuration file. The database dumps are
added to your source directories at runtime and streamed directly to
Borg. Requires the sqlite3 command. See https://sqlite.org/cli.html
for details.
example:
- name: users
path: /var/lib/db.sqlite
mongodb_databases:
type: array
items:
@@ -1749,6 +1774,9 @@ properties:
to Borg. Requires mongodump/mongorestore commands. See
https://docs.mongodb.com/database-tools/mongodump/ and
https://docs.mongodb.com/database-tools/mongorestore/ for details.
example:
- name: users
hostname: database.example.org
ntfy:
type: object
required: ['topic']
@@ -2231,9 +2259,13 @@ properties:
properties:
url:
type: string
description: URL of this Apprise service.
example: "gotify://hostname/token"
label:
type: string
description: |
Label used in borgmatic logs for this Apprise
service.
example: gotify
description: |
A list of Apprise services to publish to with URLs and
tests/unit/config/test_generate.py
-76
View File
@@ -4,82 +4,6 @@ from flexmock import flexmock
from borgmatic.config import generate as module
def test_get_properties_with_simple_object():
schema = {
'type': 'object',
'properties': dict(
[
('field1', {'example': 'Example'}),
]
),
}
assert module.get_properties(schema) == schema['properties']
def test_get_properties_merges_oneof_list_properties():
schema = {
'type': 'object',
'oneOf': [
{
'properties': dict(
[
('field1', {'example': 'Example 1'}),
('field2', {'example': 'Example 2'}),
]
),
},
{
'properties': dict(
[
('field2', {'example': 'Example 2'}),
('field3', {'example': 'Example 3'}),
]
),
},
],
}
assert module.get_properties(schema) == dict(
schema['oneOf'][0]['properties'], **schema['oneOf'][1]['properties']
)
def test_get_properties_interleaves_oneof_list_properties():
schema = {
'type': 'object',
'oneOf': [
{
'properties': dict(
[
('field1', {'example': 'Example 1'}),
('field2', {'example': 'Example 2'}),
('field3', {'example': 'Example 3'}),
]
),
},
{
'properties': dict(
[
('field4', {'example': 'Example 4'}),
('field5', {'example': 'Example 5'}),
]
),
},
],
}
assert module.get_properties(schema) == dict(
[
('field1', {'example': 'Example 1'}),
('field4', {'example': 'Example 4'}),
('field2', {'example': 'Example 2'}),
('field5', {'example': 'Example 5'}),
('field3', {'example': 'Example 3'}),
]
)
def test_schema_to_sample_configuration_generates_config_map_with_examples():
schema = {
'type': 'object',
tests/unit/config/test_schema.py
+80
View File
@@ -0,0 +1,80 @@
from borgmatic.config import schema as module
def test_get_properties_with_simple_object():
schema = {
'type': 'object',
'properties': dict(
[
('field1', {'example': 'Example'}),
]
),
}
assert module.get_properties(schema) == schema['properties']
def test_get_properties_merges_oneof_list_properties():
schema = {
'type': 'object',
'oneOf': [
{
'properties': dict(
[
('field1', {'example': 'Example 1'}),
('field2', {'example': 'Example 2'}),
]
),
},
{
'properties': dict(
[
('field2', {'example': 'Example 2'}),
('field3', {'example': 'Example 3'}),
]
),
},
],
}
assert module.get_properties(schema) == dict(
schema['oneOf'][0]['properties'], **schema['oneOf'][1]['properties']
)
def test_get_properties_interleaves_oneof_list_properties():
schema = {
'type': 'object',
'oneOf': [
{
'properties': dict(
[
('field1', {'example': 'Example 1'}),
('field2', {'example': 'Example 2'}),
('field3', {'example': 'Example 3'}),
]
),
},
{
'properties': dict(
[
('field4', {'example': 'Example 4'}),
('field5', {'example': 'Example 5'}),
]
),
},
],
}
assert module.get_properties(schema) == dict(
[
('field1', {'example': 'Example 1'}),
('field4', {'example': 'Example 4'}),
('field2', {'example': 'Example 2'}),
('field5', {'example': 'Example 5'}),
('field3', {'example': 'Example 3'}),
]
)