Bump version for release.

NEWS

@@ -1,3 +1,12 @@
+1.4.21
+ * #268: Override particular configuration options from the command-line via "--override" flag. See
+   the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/#configuration-overrides
+ * #270: Only trigger "on_error" hooks and monitoring failures for "prune", "create", and "check"
+   actions, and not for other actions.
+ * When pruning with verbosity level 1, list pruned and kept archives. Previously, this information
+   was only shown at verbosity level 2.
+
 1.4.20
  * Fix repository probing during "borgmatic init" to respect verbosity flag and remote_path option.
  * #249: Update Healthchecks/Cronitor/Cronhub monitoring integrations to fire for "check" and

borgmatic/borg/prune.py

@@ -58,7 +58,7 @@ def prune_archives(
         + (('--umask', str(umask)) if umask else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--stats',) if not dry_run and logger.isEnabledFor(logging.INFO) else ())
-        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--info', '--list') if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--dry-run',) if dry_run else ())
         + (('--stats',) if stats else ())

borgmatic/commands/arguments.py

@@ -164,6 +164,13 @@ def parse_arguments(*unparsed_arguments):
         default=None,
         help='Write log messages to this file instead of syslog',
     )
+    global_group.add_argument(
+        '--override',
+        metavar='SECTION.OPTION=VALUE',
+        nargs='+',
+        dest='overrides',
+        help='One or more configuration file options to override with specified values',
+    )
     global_group.add_argument(
         '--version',
         dest='version',

borgmatic/commands/borgmatic.py

@@ -52,9 +52,10 @@ def run_configuration(config_filename, config, arguments):
     borg_environment.initialize(storage)
     encountered_error = None
     error_repository = ''
+    prune_create_or_check = {'prune', 'create', 'check'}.intersection(arguments)
 
     try:
-        if {'prune', 'create', 'check'}.intersection(arguments):
+        if prune_create_or_check:
             dispatch.call_hooks(
                 'ping_monitor',
                 hooks,
@@ -139,7 +140,7 @@ def run_configuration(config_filename, config, arguments):
                 '{}: Error running post-backup hook'.format(config_filename), error
             )
 
-    if encountered_error:
+    if encountered_error and prune_create_or_check:
         try:
             command.execute_hook(
                 hooks.get('on_error'),
@@ -372,7 +373,7 @@ def run_actions(
                 yield json.loads(json_output)
 
 
-def load_configurations(config_filenames):
+def load_configurations(config_filenames, overrides=None):
     '''
     Given a sequence of configuration filenames, load and validate each configuration file. Return
     the results as a tuple of: dict of configuration filename to corresponding parsed configuration,
@@ -386,7 +387,7 @@ def load_configurations(config_filenames):
     for config_filename in config_filenames:
         try:
             configs[config_filename] = validate.parse_configuration(
-                config_filename, validate.schema_filename()
+                config_filename, validate.schema_filename(), overrides
             )
         except (ValueError, OSError, validate.Validation_error) as error:
             logs.extend(
@@ -584,7 +585,7 @@ def main():  # pragma: no cover
         sys.exit(0)
 
     config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
-    configs, parse_logs = load_configurations(config_filenames)
+    configs, parse_logs = load_configurations(config_filenames, global_arguments.overrides)
 
     colorama.init(autoreset=True, strip=not should_do_markup(global_arguments.no_color, configs))
     try:

borgmatic/config/override.py

@@ -0,0 +1,71 @@
+import io
+
+import ruamel.yaml
+
+
+def set_values(config, keys, value):
+    '''
+    Given a hierarchy of configuration dicts, a sequence of parsed key strings, and a string value,
+    descend into the hierarchy based on the keys to set the value into the right place.
+    '''
+    if not keys:
+        return
+
+    first_key = keys[0]
+    if len(keys) == 1:
+        config[first_key] = value
+        return
+
+    if first_key not in config:
+        config[first_key] = {}
+
+    set_values(config[first_key], keys[1:], value)
+
+
+def convert_value_type(value):
+    '''
+    Given a string value, determine its logical type (string, boolean, integer, etc.), and return it
+    converted to that type.
+    '''
+    return ruamel.yaml.YAML(typ='safe').load(io.StringIO(value))
+
+
+def parse_overrides(raw_overrides):
+    '''
+    Given a sequence of configuration file override strings in the form of "section.option=value",
+    parse and return a sequence of tuples (keys, values), where keys is a sequence of strings. For
+    instance, given the following raw overrides:
+
+        ['section.my_option=value1', 'section.other_option=value2']
+
+    ... return this:
+
+        (
+            (('section', 'my_option'), 'value1'),
+            (('section', 'other_option'), 'value2'),
+        )
+
+    Raise ValueError if an override can't be parsed.
+    '''
+    if not raw_overrides:
+        return ()
+
+    try:
+        return tuple(
+            (tuple(raw_keys.split('.')), convert_value_type(value))
+            for raw_override in raw_overrides
+            for raw_keys, value in (raw_override.split('=', 1),)
+        )
+    except ValueError:
+        raise ValueError('Invalid override. Make sure you use the form: SECTION.OPTION=VALUE')
+
+
+def apply_overrides(config, raw_overrides):
+    '''
+    Given a sequence of configuration file override strings in the form of "section.option=value"
+    and a configuration dict, parse each override and set it the configuration dict.
+    '''
+    overrides = parse_overrides(raw_overrides)
+
+    for (keys, value) in overrides:
+        set_values(config, keys, value)

borgmatic/config/validate.py

@@ -6,7 +6,7 @@ import pykwalify.core
 import pykwalify.errors
 import ruamel.yaml
 
-from borgmatic.config import load
+from borgmatic.config import load, override
 
 
 def schema_filename():
@@ -82,11 +82,12 @@ def remove_examples(schema):
     return schema
 
 
-def parse_configuration(config_filename, schema_filename):
+def parse_configuration(config_filename, schema_filename, overrides=None):
     '''
-    Given the path to a config filename in YAML format and the path to a schema filename in
-    pykwalify YAML schema format, return the parsed configuration as a data structure of nested
-    dicts and lists corresponding to the schema. Example return value:
+    Given the path to a config filename in YAML format, the path to a schema filename in pykwalify
+    YAML schema format, a sequence of configuration file override strings in the form of
+    "section.option=value", return the parsed configuration as a data structure of nested dicts and
+    lists corresponding to the schema. Example return value:
 
        {'location': {'source_directories': ['/home', '/etc'], 'repository': 'hostname.borg'},
        'retention': {'keep_daily': 7}, 'consistency': {'checks': ['repository', 'archives']}}
@@ -102,6 +103,8 @@ def parse_configuration(config_filename, schema_filename):
     except (ruamel.yaml.error.YAMLError, RecursionError) as error:
         raise Validation_error(config_filename, (str(error),))
 
+    override.apply_overrides(config, overrides)
+
     validator = pykwalify.core.Core(source_data=config, schema_data=remove_examples(schema))
     parsed_result = validator.validate(raise_exception=False)
 

docs/_includes/components/suggestion-form.html

@@ -1,12 +1,12 @@
 <h2>Improve this documentation</h2>
 
 <p>Have an idea on how to make this documentation even better? Send your
-feedback below! (But if you need help installing or using borgmatic, please
-use our <a href="https://torsion.org/borgmatic/#issues">issue tracker</a>
-instead.)</p>
+feedback below! But if you need help with borgmatic, or have an idea for a
+borgmatic feature, please use our <a href="https://torsion.org/borgmatic/#issues">issue
+tracker</a> instead.</p>
 
 <form id="suggestion-form">
-    <div><label for="suggestion">Suggestion</label></div>
+    <div><label for="suggestion">Documentation suggestion</label></div>
     <textarea id="suggestion" rows="8" cols="60" name="suggestion"></textarea>
     <div data-sk-error="suggestion" class="form-error"></div>
     <input id="_page" type="hidden" name="_page">

docs/how-to/make-per-application-backups.md

@@ -22,6 +22,11 @@ When you set up multiple configuration files like this, borgmatic will run
 each one in turn from a single borgmatic invocation. This includes, by
 default, the traditional `/etc/borgmatic/config.yaml` as well.
 
+Each configuration file is interpreted independently, as if you ran borgmatic
+for each configuration file one at a time. In other words, borgmatic does not
+perform any merging of configuration files by default. If you'd like borgmatic
+to merge your configuration files, see below about configuration includes.
+
 And if you need even more customizability, you can specify alternate
 configuration paths on the command-line with borgmatic's `--config` option.
 See `borgmatic --help` for more information.
@@ -110,6 +115,40 @@ Note that this `<<` include merging syntax is only for merging in mappings
 directly, please see the section above about standard includes.
 
 
+## Configuration overrides
+
+In more complex multi-application setups, you may want to override particular
+borgmatic configuration file options at the time you run borgmatic. For
+instance, you could reuse a common configuration file for multiple
+applications, but then set the repository for each application at runtime. Or
+you might want to try a variant of an option for testing purposes without
+actually touching your configuration file.
+
+Whatever the reason, you can override borgmatic configuration options at the
+command-line via the `--override` flag. Here's an example:
+
+```bash
+borgmatic create --override location.remote_path=borg1
+```
+
+What this does is load your configuration files, and for each one, disregard
+the configured value for the `remote_path` option in the `location` section,
+and use the value of `borg1` instead.
+
+Note that the value is parsed as an actual YAML string, so you can even set
+list values by using brackets. For instance:
+
+```bash
+borgmatic create --override location.repositories=[test1.borg,test2.borg]
+```
+
+There is not currently a way to override a single element of a list without
+replacing the whole list.
+
+Be sure to quote your overrides if they contain spaces or other characters
+that your shell may interpret.
+
+
 ## Related documentation
 
  * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)

docs/how-to/monitor-your-backups.md

@@ -57,10 +57,10 @@ tests](https://torsion.org/borgmatic/docs/how-to/extract-a-backup/).
 
 ## Error hooks
 
-When an error occurs during a backup or another action, borgmatic can run
-configurable shell commands to fire off custom error notifications or take
-other actions, so you can get alerted as soon as something goes wrong. Here's
-a not-so-useful example:
+When an error occurs during a `prune`, `create`, or `check` action, borgmatic
+can run configurable shell commands to fire off custom error notifications or
+take other actions, so you can get alerted as soon as something goes wrong.
+Here's a not-so-useful example:
 
 ```yaml
 hooks:
@@ -91,10 +91,11 @@ here:
  * `output`: output of the command that failed (may be blank if an error
    occurred without running a command)
 
-Note that borgmatic runs the `on_error` hooks for any action in which an error
-occurs, not just the `create` action. But borgmatic does not run `on_error`
-hooks if an error occurs within a `before_everything` or `after_everything`
-hook. For more about hooks, see the [borgmatic hooks
+Note that borgmatic runs the `on_error` hooks only for `prune`, `create`, or
+`check` actions or hooks in which an error occurs, and not other actions. 
+borgmatic does not run `on_error` hooks if an error occurs within a
+`before_everything` or `after_everything` hook. For more about hooks, see the
+[borgmatic hooks
 documentation](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/),
 especially the security information.
 

docs/how-to/set-up-backups.md

@@ -3,15 +3,11 @@ title: How to set up backups with borgmatic
 ---
 ## Installation
 
-To get up and running, first [install
-Borg](https://borgbackup.readthedocs.io/en/stable/installation.html), at
-least version 1.1.
+First, [install
+Borg](https://borgbackup.readthedocs.io/en/stable/installation.html), at least
+version 1.1.
 
-By default, borgmatic looks for its configuration files in `/etc/borgmatic/`
-and `/etc/borgmatic.d/`, where the root user typically has read access.
-
-So, to download and install borgmatic as the root user, run the following
-commands:
+Then, download and install borgmatic by running the following command:
 
 ```bash
 sudo pip3 install --user --upgrade borgmatic

scripts/release

@@ -23,10 +23,11 @@ git push github $version
 rm -fr dist
 python3 setup.py bdist_wheel
 python3 setup.py sdist
+gpg --detach-sign --armor dist/*
 twine upload -r pypi dist/borgmatic-*.tar.gz
 twine upload -r pypi dist/borgmatic-*-py3-none-any.whl
 
-# Set release changelogs on projects.evoworx.org and GitHub.
+# Set release changelogs on projects.torsion.org and GitHub.
 release_changelog="$(cat NEWS | sed '/^$/q' | grep -v '^\S')"
 escaped_release_changelog="$(echo "$release_changelog" | sed -z 's/\n/\\n/g' | sed -z 's/\"/\\"/g')"
 curl --silent --request POST \

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.4.20'
+VERSION = '1.4.21'
 
 
 setup(

tests/integration/config/test_override.py

@@ -0,0 +1,40 @@
+import pytest
+
+from borgmatic.config import override as module
+
+
+@pytest.mark.parametrize(
+    'value,expected_result',
+    (
+        ('thing', 'thing'),
+        ('33', 33),
+        ('33b', '33b'),
+        ('true', True),
+        ('false', False),
+        ('[foo]', ['foo']),
+        ('[foo, bar]', ['foo', 'bar']),
+    ),
+)
+def test_convert_value_type_coerces_values(value, expected_result):
+    assert module.convert_value_type(value) == expected_result
+
+
+def test_apply_overrides_updates_config():
+    raw_overrides = [
+        'section.key=value1',
+        'other_section.thing=value2',
+        'section.nested.key=value3',
+        'new.foo=bar',
+    ]
+    config = {
+        'section': {'key': 'value', 'other': 'other_value'},
+        'other_section': {'thing': 'thing_value'},
+    }
+
+    module.apply_overrides(config, raw_overrides)
+
+    assert config == {
+        'section': {'key': 'value1', 'other': 'other_value', 'nested': {'key': 'value3'}},
+        'other_section': {'thing': 'value2'},
+        'new': {'foo': 'bar'},
+    }

tests/integration/config/test_validate.py

@@ -212,3 +212,30 @@ def test_parse_configuration_raises_for_validation_error():
 
     with pytest.raises(module.Validation_error):
         module.parse_configuration('config.yaml', 'schema.yaml')
+
+
+def test_parse_configuration_applies_overrides():
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories:
+                - /home
+
+            repositories:
+                - hostname.borg
+
+            local_path: borg1
+        '''
+    )
+
+    result = module.parse_configuration(
+        'config.yaml', 'schema.yaml', overrides=['location.local_path=borg2']
+    )
+
+    assert result == {
+        'location': {
+            'source_directories': ['/home'],
+            'repositories': ['hostname.borg'],
+            'local_path': 'borg2',
+        }
+    }

tests/unit/borg/test_prune.py

@@ -75,7 +75,9 @@ def test_prune_archives_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS
     )
-    insert_execute_command_mock(PRUNE_COMMAND + ('--stats', '--info', 'repo'), logging.INFO)
+    insert_execute_command_mock(
+        PRUNE_COMMAND + ('--stats', '--info', '--list', 'repo'), logging.INFO
+    )
     insert_logging_mock(logging.INFO)
 
     module.prune_archives(

tests/unit/commands/test_borgmatic.py

@@ -119,7 +119,7 @@ def test_run_configuration_logs_on_error_hook_error():
     ).and_return(expected_results[1:])
     flexmock(module).should_receive('run_actions').and_raise(OSError)
     config = {'location': {'repositories': ['foo']}}
-    arguments = {'global': flexmock(dry_run=False)}
+    arguments = {'global': flexmock(dry_run=False), 'create': flexmock()}
 
     results = list(module.run_configuration('test.yaml', config, arguments))
 

tests/unit/config/test_override.py

@@ -0,0 +1,82 @@
+import pytest
+from flexmock import flexmock
+
+from borgmatic.config import override as module
+
+
+def test_set_values_with_empty_keys_bails():
+    config = {}
+
+    module.set_values(config, keys=(), value='value')
+
+    assert config == {}
+
+
+def test_set_values_with_one_key_sets_it_into_config():
+    config = {}
+
+    module.set_values(config, keys=('key',), value='value')
+
+    assert config == {'key': 'value'}
+
+
+def test_set_values_with_one_key_overwrites_existing_key():
+    config = {'key': 'old_value', 'other': 'other_value'}
+
+    module.set_values(config, keys=('key',), value='value')
+
+    assert config == {'key': 'value', 'other': 'other_value'}
+
+
+def test_set_values_with_multiple_keys_creates_hierarchy():
+    config = {}
+
+    module.set_values(config, ('section', 'key'), 'value')
+
+    assert config == {'section': {'key': 'value'}}
+
+
+def test_set_values_with_multiple_keys_updates_hierarchy():
+    config = {'section': {'other': 'other_value'}}
+    module.set_values(config, ('section', 'key'), 'value')
+
+    assert config == {'section': {'key': 'value', 'other': 'other_value'}}
+
+
+def test_parse_overrides_splits_keys_and_values():
+    flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value)
+    raw_overrides = ['section.my_option=value1', 'section.other_option=value2']
+    expected_result = (
+        (('section', 'my_option'), 'value1'),
+        (('section', 'other_option'), 'value2'),
+    )
+
+    module.parse_overrides(raw_overrides) == expected_result
+
+
+def test_parse_overrides_allows_value_with_equal_sign():
+    flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value)
+    raw_overrides = ['section.option=this===value']
+    expected_result = ((('section', 'option'), 'this===value'),)
+
+    module.parse_overrides(raw_overrides) == expected_result
+
+
+def test_parse_overrides_raises_on_missing_equal_sign():
+    flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value)
+    raw_overrides = ['section.option']
+
+    with pytest.raises(ValueError):
+        module.parse_overrides(raw_overrides)
+
+
+def test_parse_overrides_allows_value_with_single_key():
+    flexmock(module).should_receive('convert_value_type').replace_with(lambda value: value)
+    raw_overrides = ['option=value']
+    expected_result = ((('option',), 'value'),)
+
+    module.parse_overrides(raw_overrides) == expected_result
+
+
+def test_parse_overrides_handles_empty_overrides():
+    module.parse_overrides(raw_overrides=None) == ()