Configuration files includes and merging (#148).

NEWS

@@ -1,3 +1,11 @@
+1.3.0
+ * #148: Configuration file includes and merging via "!include" tag to support reuse of common
+   options across configuration files.
+
+1.2.18
+ * #147: Support for Borg create/extract --numeric-owner flag via "numeric_owner" option in
+   borgmatic's location section.
+
 1.2.17
  * #140: List the files within an archive via --list --archive option.
 
@@ -39,8 +47,8 @@
  * #108: Support for Borg create --progress via borgmatic command-line flag.
 
 1.2.10
- * #105: Support for Borg --chunker-params create option via "chunker_params" in borgmatic's storage
-   section.
+ * #105: Support for Borg --chunker-params create option via "chunker_params" option in borgmatic's
+   storage section.
 
 1.2.9
  * #102: Fix for syntax error that occurred in Python 3.5 and below.

borgmatic/borg/create.py

@@ -141,6 +141,7 @@ def create_archive(
         + (('--compression', compression) if compression else ())
         + (('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ())
         + (('--one-file-system',) if location_config.get('one_file_system') else ())
+        + (('--numeric-owner',) if location_config.get('numeric_owner') else ())
         + (('--read-special',) if location_config.get('read_special') else ())
         + (('--nobsdflags',) if location_config.get('bsd_flags') is False else ())
         + (('--files-cache', files_cache) if files_cache else ())

borgmatic/borg/extract.py

@@ -57,6 +57,7 @@ def extract_archive(
     repository,
     archive,
     restore_paths,
+    location_config,
     storage_config,
     local_path='borg',
     remote_path=None,
@@ -64,8 +65,8 @@ def extract_archive(
 ):
     '''
     Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to
-    restore from the archive, and a storage configuration dict, extract the archive into the current
-    directory.
+    restore from the archive, and location/storage configuration dicts, extract the archive into the
+    current directory.
     '''
     umask = storage_config.get('umask', None)
     lock_wait = storage_config.get('lock_wait', None)
@@ -74,6 +75,7 @@ def extract_archive(
         (local_path, 'extract', '::'.join((repository, archive)))
         + (tuple(restore_paths) if restore_paths else ())
         + (('--remote-path', remote_path) if remote_path else ())
+        + (('--numeric-owner',) if location_config.get('numeric_owner') else ())
         + (('--umask', str(umask)) if umask else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())

borgmatic/commands/borgmatic.py

@@ -367,6 +367,7 @@ def _run_commands_on_repository(
                 repository,
                 args.archive,
                 args.restore_paths,
+                location,
                 storage,
                 local_path=local_path,
                 remote_path=remote_path,

borgmatic/config/load.py

@@ -0,0 +1,62 @@
+import os
+import logging
+
+import ruamel.yaml
+
+
+logger = logging.getLogger(__name__)
+
+
+def load_configuration(filename):
+    '''
+    Load the given configuration file and return its contents as a data structure of nested dicts
+    and lists.
+
+    Raise ruamel.yaml.error.YAMLError if something goes wrong parsing the YAML, or RecursionError
+    if there are too many recursive includes.
+    '''
+    yaml = ruamel.yaml.YAML(typ='safe')
+    yaml.Constructor = Include_constructor
+
+    return yaml.load(open(filename))
+
+
+def include_configuration(loader, filename_node):
+    '''
+    Load the given YAML filename (ignoring the given loader so we can use our own), and return its
+    contents as a data structure of nested dicts and lists.
+    '''
+    return load_configuration(os.path.expanduser(filename_node.value))
+
+
+class Include_constructor(ruamel.yaml.SafeConstructor):
+    '''
+    A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including
+    separate YAML configuration files. Example syntax: `retention: !include common.yaml`
+    '''
+
+    def __init__(self, preserve_quotes=None, loader=None):
+        super(Include_constructor, self).__init__(preserve_quotes, loader)
+        self.add_constructor('!include', include_configuration)
+
+    def flatten_mapping(self, node):
+        '''
+        Support the special case of shallow merging included configuration into an existing mapping
+        using the YAML '<<' merge key. Example syntax:
+
+        ```
+        retention:
+            keep_daily: 1
+            <<: !include common.yaml
+        ```
+        '''
+        representer = ruamel.yaml.representer.SafeRepresenter()
+
+        for index, (key_node, value_node) in enumerate(node.value):
+            if key_node.tag == u'tag:yaml.org,2002:merge' and value_node.tag == '!include':
+                included_value = representer.represent_mapping(
+                    tag='tag:yaml.org,2002:map', mapping=self.construct_object(value_node)
+                )
+                node.value[index] = (key_node, included_value)
+
+        super(Include_constructor, self).flatten_mapping(node)

borgmatic/config/schema.yaml

@@ -32,6 +32,10 @@ map:
                 type: bool
                 desc: Stay in same file system (do not cross mount points). Defaults to false.
                 example: true
+            numeric_owner:
+                type: bool
+                desc: Only store/extract numeric user and group identifiers. Defaults to false.
+                example: true
             read_special:
                 type: bool
                 desc: |

borgmatic/config/validate.py

@@ -3,7 +3,9 @@ import logging
 import pkg_resources
 import pykwalify.core
 import pykwalify.errors
-from ruamel import yaml
+import ruamel.yaml
+
+from borgmatic.config import load
 
 
 logger = logging.getLogger(__name__)
@@ -87,9 +89,9 @@ def parse_configuration(config_filename, schema_filename):
     logging.getLogger('pykwalify').setLevel(logging.ERROR)
 
     try:
-        config = yaml.safe_load(open(config_filename))
-        schema = yaml.safe_load(open(schema_filename))
-    except yaml.error.YAMLError as error:
+        config = load.load_configuration(config_filename)
+        schema = load.load_configuration(schema_filename)
+    except (ruamel.yaml.error.YAMLError, RecursionError) as error:
         raise Validation_error(config_filename, (str(error),))
 
     # pykwalify gets angry if the example field is not a string. So rather than bend to its will,

docs/how-to/make-per-application-backups.md

@@ -27,6 +27,85 @@ configuration paths on the command-line with borgmatic's `--config` option.
 See `borgmatic --help` for more information.
 
 
+## Configuration includes
+
+Once you have multiple different configuration files, you might want to share
+common configuration options across these files with having to copy and paste
+them. To achieve this, you can put fragments of common configuration options
+into a file, and then include or inline that file into one or more borgmatic
+configuration files.
+
+Let's say that you want to include common retention configuration across all
+of your configuration files. You could do that in each configuration file with
+the following:
+
+```yaml
+location:
+   ...
+
+retention:
+    !include /etc/borgmatic/common_retention.yaml
+```
+
+And then the contents of `common_retention.yaml` could be:
+
+```yaml
+keep_hourly: 24
+keep_daily: 7
+```
+
+To prevent borgmatic from trying to load these configuration fragments by
+themselves and complaining that they are not valid configuration files, you
+should put them in a directory other than `/etc/borgmatic.d/`. (A subdirectory
+is fine.)
+
+Note that this form of include must be a YAML value rather than a key. For
+example, this will not work:
+
+```yaml
+location:
+   ...
+
+# Don't do this. It won't work!
+!include /etc/borgmatic/common_retention.yaml
+```
+
+But if you do want to merge in a YAML key and its values, keep reading!
+
+
+## Include merging
+
+If you need to get even fancier and pull in common configuration options while
+potentially overriding individual options, you can perform a YAML merge of
+included configuration using the YAML `<<` key. For instance, here's an
+example of a main configuration file that pulls in two retention options via
+an include, and then overrides one of them locally:
+
+```yaml
+location:
+   ...
+
+retention:
+    keep_daily: 5
+    <<: !include /etc/borgmatic/common_retention.yaml
+```
+
+This is what `common_retention.yaml` might look like:
+
+```yaml
+keep_hourly: 24
+keep_daily: 7
+```
+
+Once this include gets merged in, the resulting configuration would have a
+`keep_hourly` value of `24` and an overridden `keep_daily` value of `5`.
+
+When there is a collision of an option between the local file and the merged
+include, the local file's option takes precedent. And note that this is a
+shallow merge rather than a deep merge, so the merging does not descend into
+nested values.
+
+
 ## Related documentation
 
  * [Set up backups with borgmatic](../../docs/how-to/set-up-backups.md)

docs/how-to/set-up-backups.md

@@ -45,9 +45,14 @@ not in your system `PATH`. Try looking in `/usr/local/bin/`.
 
 This generates a sample configuration file at /etc/borgmatic/config.yaml (by
 default). You should edit the file to suit your needs, as the values are
-representative. All fields are optional except where indicated, so feel free
+representative. All options are optional except where indicated, so feel free
 to ignore anything you don't need.
 
+Note that the configuration file is organized into distinct sections, each
+with a section name like `location:` or `storage:`. So take care that if you
+uncomment a particular option, also uncomment its containing section name, or
+else borgmatic won't recognize the option.
+
 You can also get the same sample configuration file from the [configuration
 reference](../../docs/reference/configuration.md), the authoritative set of
 all configuration options. This is handy if borgmatic has added new options
@@ -71,6 +76,9 @@ FAQ](http://borgbackup.readthedocs.io/en/stable/faq.html#how-can-i-specify-the-e
 for more info.
 
 
+###
+
+
 ## Initialization
 
 Before you can create backups with borgmatic, you first need to initialize a

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.2.17'
+VERSION = '1.3.0'
 
 
 setup(

tests/integration/config/test_load.py

@@ -0,0 +1,40 @@
+import sys
+
+from flexmock import flexmock
+
+from borgmatic.config import load as module
+
+
+def test_load_configuration_parses_contents():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('config.yaml').and_return('key: value')
+
+    assert module.load_configuration('config.yaml') == {'key': 'value'}
+
+
+def test_load_configuration_inlines_include():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return('value')
+    builtins.should_receive('open').with_args('config.yaml').and_return(
+        'key: !include include.yaml'
+    )
+
+    assert module.load_configuration('config.yaml') == {'key': 'value'}
+
+
+def test_load_configuration_merges_include():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return(
+        '''
+        foo: bar
+        baz: quux
+        '''
+    )
+    builtins.should_receive('open').with_args('config.yaml').and_return(
+        '''
+        foo: override
+        <<: !include include.yaml
+        '''
+    )
+
+    assert module.load_configuration('config.yaml') == {'foo': 'override', 'baz': 'quux'}

tests/integration/config/test_validate.py

@@ -118,6 +118,67 @@ def test_parse_configuration_with_schema_lacking_examples_does_not_raise():
     module.parse_configuration('config.yaml', 'schema.yaml')
 
 
+def test_parse_configuration_inlines_include():
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories:
+                - /home
+
+            repositories:
+                - hostname.borg
+
+        retention:
+            !include include.yaml
+        '''
+    )
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return(
+        '''
+        keep_daily: 7
+        keep_hourly: 24
+        '''
+    )
+
+    result = module.parse_configuration('config.yaml', 'schema.yaml')
+
+    assert result == {
+        'location': {'source_directories': ['/home'], 'repositories': ['hostname.borg']},
+        'retention': {'keep_daily': 7, 'keep_hourly': 24},
+    }
+
+
+def test_parse_configuration_merges_include():
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories:
+                - /home
+
+            repositories:
+                - hostname.borg
+
+        retention:
+            keep_daily: 1
+            <<: !include include.yaml
+        '''
+    )
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return(
+        '''
+        keep_daily: 7
+        keep_hourly: 24
+        '''
+    )
+
+    result = module.parse_configuration('config.yaml', 'schema.yaml')
+
+    assert result == {
+        'location': {'source_directories': ['/home'], 'repositories': ['hostname.borg']},
+        'retention': {'keep_daily': 1, 'keep_hourly': 24},
+    }
+
+
 def test_parse_configuration_raises_for_missing_config_file():
     with pytest.raises(FileNotFoundError):
         module.parse_configuration('config.yaml', 'schema.yaml')

tests/unit/borg/test_create.py

@@ -425,7 +425,7 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
     )
 
 
-def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameters():
+def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameter():
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -446,6 +446,27 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
     )
 
 
+def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_parameter():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND + ('--numeric-owner',))
+
+    module.create_archive(
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'numeric_owner': True,
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
 def test_create_archive_with_read_special_calls_borg_with_read_special_parameter():
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())

tests/unit/borg/test_extract.py

@@ -111,6 +111,7 @@ def test_extract_archive_calls_borg_with_restore_path_parameters():
         repository='repo',
         archive='archive',
         restore_paths=['path1', 'path2'],
+        location_config={},
         storage_config={},
     )
 
@@ -123,11 +124,25 @@ def test_extract_archive_calls_borg_with_remote_path_parameters():
         repository='repo',
         archive='archive',
         restore_paths=None,
+        location_config={},
         storage_config={},
         remote_path='borg1',
     )
 
 
+def test_extract_archive_calls_borg_with_numeric_owner_parameter():
+    insert_subprocess_mock(('borg', 'extract', 'repo::archive', '--numeric-owner'))
+
+    module.extract_archive(
+        dry_run=False,
+        repository='repo',
+        archive='archive',
+        restore_paths=None,
+        location_config={'numeric_owner': True},
+        storage_config={},
+    )
+
+
 def test_extract_archive_calls_borg_with_umask_parameters():
     insert_subprocess_mock(('borg', 'extract', 'repo::archive', '--umask', '0770'))
 
@@ -136,6 +151,7 @@ def test_extract_archive_calls_borg_with_umask_parameters():
         repository='repo',
         archive='archive',
         restore_paths=None,
+        location_config={},
         storage_config={'umask': '0770'},
     )
 
@@ -148,6 +164,7 @@ def test_extract_archive_calls_borg_with_lock_wait_parameters():
         repository='repo',
         archive='archive',
         restore_paths=None,
+        location_config={},
         storage_config={'lock_wait': '5'},
     )
 
@@ -157,7 +174,12 @@ def test_extract_archive_with_log_info_calls_borg_with_info_parameter():
     insert_logging_mock(logging.INFO)
 
     module.extract_archive(
-        dry_run=False, repository='repo', archive='archive', restore_paths=None, storage_config={}
+        dry_run=False,
+        repository='repo',
+        archive='archive',
+        restore_paths=None,
+        location_config={},
+        storage_config={},
     )
 
 
@@ -166,7 +188,12 @@ def test_extract_archive_with_log_debug_calls_borg_with_debug_parameters():
     insert_logging_mock(logging.DEBUG)
 
     module.extract_archive(
-        dry_run=False, repository='repo', archive='archive', restore_paths=None, storage_config={}
+        dry_run=False,
+        repository='repo',
+        archive='archive',
+        restore_paths=None,
+        location_config={},
+        storage_config={},
     )
 
 
@@ -174,7 +201,12 @@ def test_extract_archive_calls_borg_with_dry_run_parameter():
     insert_subprocess_mock(('borg', 'extract', 'repo::archive', '--dry-run'))
 
     module.extract_archive(
-        dry_run=True, repository='repo', archive='archive', restore_paths=None, storage_config={}
+        dry_run=True,
+        repository='repo',
+        archive='archive',
+        restore_paths=None,
+        location_config={},
+        storage_config={},
     )
 
 
@@ -186,6 +218,7 @@ def test_extract_archive_calls_borg_with_progress_parameter():
         repository='repo',
         archive='archive',
         restore_paths=None,
+        location_config={},
         storage_config={},
         progress=True,
     )