Fix for invalid JSON output when using multiple borgmatic configuration files (#155).

NEWS

@@ -1,3 +1,22 @@
+1.3.1
+ * #155: Fix for invalid JSON output when using multiple borgmatic configuration files.
+ * #157: Fix for seemingly random filename ordering when running through a directory of
+   configuration files.
+ * Fix for empty JSON output when using --create --json.
+ * Now capturing Borg output only when --json flag is used. Previously, borgmatic delayed Borg
+   output even without the --json flag.
+
+1.3.0
+ * #148: Configuration file includes and merging via "!include" tag to support reuse of common
+   options across configuration files.
+
+1.2.18
+ * #147: Support for Borg create/extract --numeric-owner flag via "numeric_owner" option in
+   borgmatic's location section.
+
+1.2.17
+ * #140: List the files within an archive via --list --archive option.
+
 1.2.16
  * #119: Include a sample borgmatic configuration file in the documentation.
  * #123: Support for Borg archive restoration via borgmatic --extract command-line flag.
@@ -36,8 +55,8 @@
  * #108: Support for Borg create --progress via borgmatic command-line flag.
 
 1.2.10
- * #105: Support for Borg --chunker-params create option via "chunker_params" in borgmatic's storage
-   section.
+ * #105: Support for Borg --chunker-params create option via "chunker_params" option in borgmatic's
+   storage section.
 
 1.2.9
  * #102: Fix for syntax error that occurred in Python 3.5 and below.

borgmatic/borg/create.py

@@ -2,9 +2,10 @@ import glob
 import itertools
 import logging
 import os
-import subprocess
 import tempfile
 
+from borgmatic.borg.execute import execute_command
+
 
 logger = logging.getLogger(__name__)
 
@@ -107,7 +108,7 @@ def create_archive(
 ):
     '''
     Given vebosity/dry-run flags, a local or remote repository path, a location config dict, and a
-    storage config dict, create a Borg archive.
+    storage config dict, create a Borg archive and return Borg's JSON output (if any).
     '''
     sources = _expand_directories(location_config['source_directories'])
 
@@ -141,6 +142,7 @@ def create_archive(
         + (('--compression', compression) if compression else ())
         + (('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ())
         + (('--one-file-system',) if location_config.get('one_file_system') else ())
+        + (('--numeric-owner',) if location_config.get('numeric_owner') else ())
         + (('--read-special',) if location_config.get('read_special') else ())
         + (('--nobsdflags',) if location_config.get('bsd_flags') is False else ())
         + (('--files-cache', files_cache) if files_cache else ())
@@ -156,5 +158,4 @@ def create_archive(
         + (('--json',) if json else ())
     )
 
-    logger.debug(' '.join(full_command))
-    subprocess.check_call(full_command)
+    return execute_command(full_command, capture_output=json)

borgmatic/borg/execute.py

@@ -0,0 +1,19 @@
+import logging
+import subprocess
+
+
+logger = logging.getLogger(__name__)
+
+
+def execute_command(full_command, capture_output=False):
+    '''
+    Execute the given command (a sequence of command/argument strings). If capture output is True,
+    then return the command's output as a string.
+    '''
+    logger.debug(' '.join(full_command))
+
+    if capture_output:
+        output = subprocess.check_output(full_command)
+        return output.decode() if output is not None else None
+    else:
+        subprocess.check_call(full_command)

borgmatic/borg/extract.py

@@ -57,6 +57,7 @@ def extract_archive(
     repository,
     archive,
     restore_paths,
+    location_config,
     storage_config,
     local_path='borg',
     remote_path=None,
@@ -64,8 +65,8 @@ def extract_archive(
 ):
     '''
     Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to
-    restore from the archive, and a storage configuration dict, extract the archive into the current
-    directory.
+    restore from the archive, and location/storage configuration dicts, extract the archive into the
+    current directory.
     '''
     umask = storage_config.get('umask', None)
     lock_wait = storage_config.get('lock_wait', None)
@@ -74,6 +75,7 @@ def extract_archive(
         (local_path, 'extract', '::'.join((repository, archive)))
         + (tuple(restore_paths) if restore_paths else ())
         + (('--remote-path', remote_path) if remote_path else ())
+        + (('--numeric-owner',) if location_config.get('numeric_owner') else ())
         + (('--umask', str(umask)) if umask else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())

borgmatic/borg/info.py

@@ -1,5 +1,6 @@
 import logging
-import subprocess
+
+from borgmatic.borg.execute import execute_command
 
 
 logger = logging.getLogger(__name__)
@@ -9,8 +10,8 @@ def display_archives_info(
     repository, storage_config, local_path='borg', remote_path=None, json=False
 ):
     '''
-    Given a local or remote repository path, and a storage config dict,
-    display summary information for Borg archives in the repository.
+    Given a local or remote repository path, and a storage config dict, display summary information
+    for Borg archives in the repository or return JSON summary information.
     '''
     lock_wait = storage_config.get('lock_wait', None)
 
@@ -23,7 +24,4 @@ def display_archives_info(
         + (('--json',) if json else ())
     )
 
-    logger.debug(' '.join(full_command))
-
-    output = subprocess.check_output(full_command)
-    return output.decode() if output is not None else None
+    return execute_command(full_command, capture_output=json)

borgmatic/borg/list.py

@@ -1,26 +1,28 @@
 import logging
-import subprocess
+
+from borgmatic.borg.execute import execute_command
 
 
 logger = logging.getLogger(__name__)
 
 
-def list_archives(repository, storage_config, local_path='borg', remote_path=None, json=False):
+def list_archives(
+    repository, storage_config, archive=None, local_path='borg', remote_path=None, json=False
+):
     '''
-    Given a local or remote repository path, and a storage config dict,
-    list Borg archives in the repository.
+    Given a local or remote repository path and a storage config dict, display the output of listing
+    Borg archives in the repository or return JSON output. Or, if an archive name is given, listing
+    the files in that archive.
     '''
     lock_wait = storage_config.get('lock_wait', None)
 
     full_command = (
-        (local_path, 'list', repository)
+        (local_path, 'list', '::'.join((repository, archive)) if archive else repository)
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--json',) if json else ())
     )
-    logger.debug(' '.join(full_command))
 
-    output = subprocess.check_output(full_command)
-    return output.decode() if output is not None else None
+    return execute_command(full_command, capture_output=json)

borgmatic/commands/borgmatic.py

@@ -102,27 +102,49 @@ def parse_arguments(*arguments):
         help='Create a repository with a fixed storage quota',
     )
 
+    prune_group = parser.add_argument_group('options for --prune')
+    stats_argument = prune_group.add_argument(
+        '--stats',
+        dest='stats',
+        default=False,
+        action='store_true',
+        help='Display statistics of archive',
+    )
+
     create_group = parser.add_argument_group('options for --create')
-    create_group.add_argument(
+    progress_argument = create_group.add_argument(
         '--progress',
         dest='progress',
         default=False,
         action='store_true',
-        help='Display progress for each file as it is backed up',
+        help='Display progress for each file as it is processed',
+    )
+    create_group._group_actions.append(stats_argument)
+    json_argument = create_group.add_argument(
+        '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
     )
 
     extract_group = parser.add_argument_group('options for --extract')
-    extract_group.add_argument(
+    repository_argument = extract_group.add_argument(
         '--repository',
-        help='Path of repository to restore from, defaults to the configured repository if there is only one',
+        help='Path of repository to use, defaults to the configured repository if there is only one',
     )
-    extract_group.add_argument('--archive', help='Name of archive to restore')
+    archive_argument = extract_group.add_argument('--archive', help='Name of archive to operate on')
     extract_group.add_argument(
         '--restore-path',
         nargs='+',
         dest='restore_paths',
         help='Paths to restore from archive, defaults to the entire archive',
     )
+    extract_group._group_actions.append(progress_argument)
+
+    list_group = parser.add_argument_group('options for --list')
+    list_group._group_actions.append(repository_argument)
+    list_group._group_actions.append(archive_argument)
+    list_group._group_actions.append(json_argument)
+
+    info_group = parser.add_argument_group('options for --info')
+    info_group._group_actions.append(json_argument)
 
     common_group = parser.add_argument_group('common options')
     common_group.add_argument(
@@ -140,20 +162,6 @@ def parse_arguments(*arguments):
         dest='excludes_filename',
         help='Deprecated in favor of exclude_patterns within configuration',
     )
-    common_group.add_argument(
-        '--stats',
-        dest='stats',
-        default=False,
-        action='store_true',
-        help='Display statistics of archive with --create or --prune option',
-    )
-    common_group.add_argument(
-        '--json',
-        dest='json',
-        default=False,
-        action='store_true',
-        help='Output results from the --create, --list, or --info options as json',
-    )
     common_group.add_argument(
         '-n',
         '--dry-run',
@@ -196,10 +204,15 @@ def parse_arguments(*arguments):
         raise ValueError('The --encryption option is required with the --init option')
 
     if not args.extract:
-        if args.repository:
-            raise ValueError('The --repository option can only be used with the --extract option')
-        if args.archive:
-            raise ValueError('The --archive option can only be used with the --extract option')
+        if not args.list:
+            if args.repository:
+                raise ValueError(
+                    'The --repository option can only be used with the --extract and --list options'
+                )
+            if args.archive:
+                raise ValueError(
+                    'The --archive option can only be used with the --extract and --list options'
+                )
         if args.restore_paths:
             raise ValueError('The --restore-path option can only be used with the --extract option')
     if args.extract and not args.archive:
@@ -245,6 +258,8 @@ def run_configuration(config_filename, config, args):  # pragma: no cover
     '''
     Given a config filename and the corresponding parsed config dict, execute its defined pruning,
     backups, consistency checks, and/or other actions.
+
+    Yield JSON output strings from executing any actions that produce JSON.
     '''
     (location, storage, retention, consistency, hooks) = (
         config.get(section_name, {})
@@ -259,15 +274,17 @@ def run_configuration(config_filename, config, args):  # pragma: no cover
         if args.create:
             hook.execute_hook(hooks.get('before_backup'), config_filename, 'pre-backup')
 
-        _run_commands(
-            args=args,
-            consistency=consistency,
-            local_path=local_path,
-            location=location,
-            remote_path=remote_path,
-            retention=retention,
-            storage=storage,
-        )
+        for repository_path in location['repositories']:
+            yield from run_actions(
+                args=args,
+                location=location,
+                storage=storage,
+                retention=retention,
+                consistency=consistency,
+                local_path=local_path,
+                remote_path=remote_path,
+                repository_path=repository_path,
+            )
 
         if args.create:
             hook.execute_hook(hooks.get('after_backup'), config_filename, 'post-backup')
@@ -276,37 +293,17 @@ def run_configuration(config_filename, config, args):  # pragma: no cover
         raise
 
 
-def _run_commands(*, args, consistency, local_path, location, remote_path, retention, storage):
-    json_results = []
-    for unexpanded_repository in location['repositories']:
-        _run_commands_on_repository(
-            args=args,
-            consistency=consistency,
-            json_results=json_results,
-            local_path=local_path,
-            location=location,
-            remote_path=remote_path,
-            retention=retention,
-            storage=storage,
-            unexpanded_repository=unexpanded_repository,
-        )
-    if args.json:
-        sys.stdout.write(json.dumps(json_results))
-
-
-def _run_commands_on_repository(
-    *,
-    args,
-    consistency,
-    json_results,
-    local_path,
-    location,
-    remote_path,
-    retention,
-    storage,
-    unexpanded_repository
+def run_actions(
+    *, args, location, storage, retention, consistency, local_path, remote_path, repository_path
 ):  # pragma: no cover
-    repository = os.path.expanduser(unexpanded_repository)
+    '''
+    Given parsed command-line arguments as an argparse.ArgumentParser instance, several different
+    configuration dicts, local and remote paths to Borg, and a repository name, run all actions
+    from the command-line arguments on the given repository.
+
+    Yield JSON output strings from executing any actions that produce JSON.
+    '''
+    repository = os.path.expanduser(repository_path)
     dry_run_label = ' (dry run; not making any changes)' if args.dry_run else ''
     if args.init:
         logger.info('{}: Initializing repository'.format(repository))
@@ -331,7 +328,7 @@ def _run_commands_on_repository(
         )
     if args.create:
         logger.info('{}: Creating archive{}'.format(repository, dry_run_label))
-        borg_create.create_archive(
+        json_output = borg_create.create_archive(
             args.dry_run,
             repository,
             location,
@@ -340,7 +337,10 @@ def _run_commands_on_repository(
             remote_path=remote_path,
             progress=args.progress,
             stats=args.stats,
+            json=args.json,
         )
+        if json_output:
+            yield json.loads(json_output)
     if args.check and checks.repository_enabled_for_checks(repository, consistency):
         logger.info('{}: Running consistency checks'.format(repository))
         borg_check.check_archives(
@@ -354,29 +354,32 @@ def _run_commands_on_repository(
                 repository,
                 args.archive,
                 args.restore_paths,
+                location,
                 storage,
                 local_path=local_path,
                 remote_path=remote_path,
                 progress=args.progress,
             )
     if args.list:
-        logger.info('{}: Listing archives'.format(repository))
-        output = borg_list.list_archives(
-            repository, storage, local_path=local_path, remote_path=remote_path, json=args.json
-        )
-        if args.json:
-            json_results.append(json.loads(output))
-        else:
-            sys.stdout.write(output)
+        if args.repository is None or repository == args.repository:
+            logger.info('{}: Listing archives'.format(repository))
+            json_output = borg_list.list_archives(
+                repository,
+                storage,
+                args.archive,
+                local_path=local_path,
+                remote_path=remote_path,
+                json=args.json,
+            )
+            if json_output:
+                yield json.loads(json_output)
     if args.info:
         logger.info('{}: Displaying summary info for archives'.format(repository))
-        output = borg_info.display_archives_info(
+        json_output = borg_info.display_archives_info(
             repository, storage, local_path=local_path, remote_path=remote_path, json=args.json
         )
-        if args.json:
-            json_results.append(json.loads(output))
-        else:
-            sys.stdout.write(output)
+        if json_output:
+            yield json.loads(json_output)
 
 
 def collect_configuration_run_summary_logs(config_filenames, args):
@@ -384,10 +387,14 @@ def collect_configuration_run_summary_logs(config_filenames, args):
     Given a sequence of configuration filenames and parsed command-line arguments as an
     argparse.ArgumentParser instance, run each configuration file and yield a series of
     logging.LogRecord instances containing summary information about each run.
+
+    As a side effect of running through these configuration files, output their JSON results, if
+    any, to stdout.
     '''
     # Dict mapping from config filename to corresponding parsed config dict.
     configs = collections.OrderedDict()
 
+    # Parse and load each configuration file.
     for config_filename in config_filenames:
         try:
             logger.info('{}: Parsing configuration file'.format(config_filename))
@@ -403,16 +410,19 @@ def collect_configuration_run_summary_logs(config_filenames, args):
             )
             yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
 
-    if args.extract:
+    # Run cross-file validation checks.
+    if args.extract or (args.list and args.archive):
         try:
             validate.guard_configuration_contains_repository(args.repository, configs)
         except ValueError as error:
             yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
             return
 
+    # Execute the actions corresponding to each configuration file.
+    json_results = []
     for config_filename, config in configs.items():
         try:
-            run_configuration(config_filename, config, args)
+            json_results.extend(list(run_configuration(config_filename, config, args)))
             yield logging.makeLogRecord(
                 dict(
                     levelno=logging.INFO,
@@ -428,6 +438,9 @@ def collect_configuration_run_summary_logs(config_filenames, args):
             )
             yield logging.makeLogRecord(dict(levelno=logging.CRITICAL, msg=error))
 
+    if json_results:
+        sys.stdout.write(json.dumps(json_results))
+
     if not config_filenames:
         yield logging.makeLogRecord(
             dict(

borgmatic/config/collect.py

@@ -41,7 +41,7 @@ def collect_config_filenames(config_paths):
             yield path
             continue
 
-        for filename in os.listdir(path):
+        for filename in sorted(os.listdir(path)):
             full_filename = os.path.join(path, filename)
             if full_filename.endswith('.yaml') and not os.path.isdir(full_filename):
                 yield full_filename

borgmatic/config/load.py

@@ -0,0 +1,62 @@
+import os
+import logging
+
+import ruamel.yaml
+
+
+logger = logging.getLogger(__name__)
+
+
+def load_configuration(filename):
+    '''
+    Load the given configuration file and return its contents as a data structure of nested dicts
+    and lists.
+
+    Raise ruamel.yaml.error.YAMLError if something goes wrong parsing the YAML, or RecursionError
+    if there are too many recursive includes.
+    '''
+    yaml = ruamel.yaml.YAML(typ='safe')
+    yaml.Constructor = Include_constructor
+
+    return yaml.load(open(filename))
+
+
+def include_configuration(loader, filename_node):
+    '''
+    Load the given YAML filename (ignoring the given loader so we can use our own), and return its
+    contents as a data structure of nested dicts and lists.
+    '''
+    return load_configuration(os.path.expanduser(filename_node.value))
+
+
+class Include_constructor(ruamel.yaml.SafeConstructor):
+    '''
+    A YAML "constructor" (a ruamel.yaml concept) that supports a custom "!include" tag for including
+    separate YAML configuration files. Example syntax: `retention: !include common.yaml`
+    '''
+
+    def __init__(self, preserve_quotes=None, loader=None):
+        super(Include_constructor, self).__init__(preserve_quotes, loader)
+        self.add_constructor('!include', include_configuration)
+
+    def flatten_mapping(self, node):
+        '''
+        Support the special case of shallow merging included configuration into an existing mapping
+        using the YAML '<<' merge key. Example syntax:
+
+        ```
+        retention:
+            keep_daily: 1
+            <<: !include common.yaml
+        ```
+        '''
+        representer = ruamel.yaml.representer.SafeRepresenter()
+
+        for index, (key_node, value_node) in enumerate(node.value):
+            if key_node.tag == u'tag:yaml.org,2002:merge' and value_node.tag == '!include':
+                included_value = representer.represent_mapping(
+                    tag='tag:yaml.org,2002:map', mapping=self.construct_object(value_node)
+                )
+                node.value[index] = (key_node, included_value)
+
+        super(Include_constructor, self).flatten_mapping(node)

borgmatic/config/schema.yaml

@@ -32,6 +32,10 @@ map:
                 type: bool
                 desc: Stay in same file system (do not cross mount points). Defaults to false.
                 example: true
+            numeric_owner:
+                type: bool
+                desc: Only store/extract numeric user and group identifiers. Defaults to false.
+                example: true
             read_special:
                 type: bool
                 desc: |

borgmatic/config/validate.py

@@ -3,7 +3,9 @@ import logging
 import pkg_resources
 import pykwalify.core
 import pykwalify.errors
-from ruamel import yaml
+import ruamel.yaml
+
+from borgmatic.config import load
 
 
 logger = logging.getLogger(__name__)
@@ -87,9 +89,9 @@ def parse_configuration(config_filename, schema_filename):
     logging.getLogger('pykwalify').setLevel(logging.ERROR)
 
     try:
-        config = yaml.safe_load(open(config_filename))
-        schema = yaml.safe_load(open(schema_filename))
-    except yaml.error.YAMLError as error:
+        config = load.load_configuration(config_filename)
+        schema = load.load_configuration(schema_filename)
+    except (ruamel.yaml.error.YAMLError, RecursionError) as error:
         raise Validation_error(config_filename, (str(error),))
 
     # pykwalify gets angry if the example field is not a string. So rather than bend to its will,
@@ -130,7 +132,7 @@ def guard_configuration_contains_repository(repository, configurations):
 
         if count > 1:
             raise ValueError(
-                'Can\'t determine which repository to extract. Use --repository option to disambiguate'.format(
+                'Can\'t determine which repository to use. Use --repository option to disambiguate'.format(
                     repository
                 )
             )

docs/how-to/make-per-application-backups.md

@@ -27,6 +27,85 @@ configuration paths on the command-line with borgmatic's `--config` option.
 See `borgmatic --help` for more information.
 
 
+## Configuration includes
+
+Once you have multiple different configuration files, you might want to share
+common configuration options across these files with having to copy and paste
+them. To achieve this, you can put fragments of common configuration options
+into a file, and then include or inline that file into one or more borgmatic
+configuration files.
+
+Let's say that you want to include common retention configuration across all
+of your configuration files. You could do that in each configuration file with
+the following:
+
+```yaml
+location:
+   ...
+
+retention:
+    !include /etc/borgmatic/common_retention.yaml
+```
+
+And then the contents of `common_retention.yaml` could be:
+
+```yaml
+keep_hourly: 24
+keep_daily: 7
+```
+
+To prevent borgmatic from trying to load these configuration fragments by
+themselves and complaining that they are not valid configuration files, you
+should put them in a directory other than `/etc/borgmatic.d/`. (A subdirectory
+is fine.)
+
+Note that this form of include must be a YAML value rather than a key. For
+example, this will not work:
+
+```yaml
+location:
+   ...
+
+# Don't do this. It won't work!
+!include /etc/borgmatic/common_retention.yaml
+```
+
+But if you do want to merge in a YAML key and its values, keep reading!
+
+
+## Include merging
+
+If you need to get even fancier and pull in common configuration options while
+potentially overriding individual options, you can perform a YAML merge of
+included configuration using the YAML `<<` key. For instance, here's an
+example of a main configuration file that pulls in two retention options via
+an include, and then overrides one of them locally:
+
+```yaml
+location:
+   ...
+
+retention:
+    keep_daily: 5
+    <<: !include /etc/borgmatic/common_retention.yaml
+```
+
+This is what `common_retention.yaml` might look like:
+
+```yaml
+keep_hourly: 24
+keep_daily: 7
+```
+
+Once this include gets merged in, the resulting configuration would have a
+`keep_hourly` value of `24` and an overridden `keep_daily` value of `5`.
+
+When there is a collision of an option between the local file and the merged
+include, the local file's option takes precedent. And note that this is a
+shallow merge rather than a deep merge, so the merging does not descend into
+nested values.
+
+
 ## Related documentation
 
  * [Set up backups with borgmatic](../../docs/how-to/set-up-backups.md)

docs/how-to/set-up-backups.md

@@ -22,8 +22,8 @@ you're using Python 3, as borgmatic does not support Python 2.
 Along with the above process, you have several other options for installing
 borgmatic:
 
- * [Docker image](https://hub.docker.com/r/monachus/borgmatic/)
- * [Another Docker image](https://hub.docker.com/r/b3vis/borgmatic/)
+ * [Docker base image](https://hub.docker.com/r/monachus/borgmatic/)
+ * [Docker image with support for scheduled backups](https://hub.docker.com/r/b3vis/borgmatic/)
  * [Debian](https://tracker.debian.org/pkg/borgmatic)
  * [Ubuntu](https://launchpad.net/ubuntu/+source/borgmatic)
  * [Fedora](https://bodhi.fedoraproject.org/updates/?search=borgmatic)
@@ -45,9 +45,14 @@ not in your system `PATH`. Try looking in `/usr/local/bin/`.
 
 This generates a sample configuration file at /etc/borgmatic/config.yaml (by
 default). You should edit the file to suit your needs, as the values are
-representative. All fields are optional except where indicated, so feel free
+representative. All options are optional except where indicated, so feel free
 to ignore anything you don't need.
 
+Note that the configuration file is organized into distinct sections, each
+with a section name like `location:` or `storage:`. So take care that if you
+uncomment a particular option, also uncomment its containing section name, or
+else borgmatic won't recognize the option.
+
 You can also get the same sample configuration file from the [configuration
 reference](../../docs/reference/configuration.md), the authoritative set of
 all configuration options. This is handy if borgmatic has added new options
@@ -71,6 +76,9 @@ FAQ](http://borgbackup.readthedocs.io/en/stable/faq.html#how-can-i-specify-the-e
 for more info.
 
 
+###
+
+
 ## Initialization
 
 Before you can create backups with borgmatic, you first need to initialize a

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.2.16'
+VERSION = '1.3.1'
 
 
 setup(

tests/integration/commands/test_borgmatic.py

@@ -142,14 +142,28 @@ def test_parse_arguments_disallows_init_and_dry_run():
         )
 
 
-def test_parse_arguments_disallows_repository_without_extract():
+def test_parse_arguments_disallows_repository_without_extract_or_list():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 
     with pytest.raises(ValueError):
         module.parse_arguments('--config', 'myconfig', '--repository', 'test.borg')
 
 
-def test_parse_arguments_disallows_archive_without_extract():
+def test_parse_arguments_allows_repository_with_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments(
+        '--config', 'myconfig', '--extract', '--repository', 'test.borg', '--archive', 'test'
+    )
+
+
+def test_parse_arguments_allows_repository_with_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', '--list', '--repository', 'test.borg')
+
+
+def test_parse_arguments_disallows_archive_without_extract_or_list():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 
     with pytest.raises(ValueError):
@@ -169,6 +183,12 @@ def test_parse_arguments_allows_archive_with_extract():
     module.parse_arguments('--config', 'myconfig', '--extract', '--archive', 'test')
 
 
+def test_parse_arguments_allows_archive_with_list():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
+    module.parse_arguments('--config', 'myconfig', '--list', '--archive', 'test')
+
+
 def test_parse_arguments_requires_archive_with_extract():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 
@@ -177,51 +197,73 @@ def test_parse_arguments_requires_archive_with_extract():
 
 
 def test_parse_arguments_allows_progress_and_create():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
     module.parse_arguments('--progress', '--create', '--list')
 
 
 def test_parse_arguments_allows_progress_and_extract():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
     module.parse_arguments('--progress', '--extract', '--archive', 'test', '--list')
 
 
 def test_parse_arguments_disallows_progress_without_create():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
     with pytest.raises(ValueError):
         module.parse_arguments('--progress', '--list')
 
 
 def test_parse_arguments_with_stats_and_create_flags_does_not_raise():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
     module.parse_arguments('--stats', '--create', '--list')
 
 
 def test_parse_arguments_with_stats_and_prune_flags_does_not_raise():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
     module.parse_arguments('--stats', '--prune', '--list')
 
 
 def test_parse_arguments_with_stats_flag_but_no_create_or_prune_flag_raises_value_error():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
     with pytest.raises(ValueError):
         module.parse_arguments('--stats', '--list')
 
 
 def test_parse_arguments_with_just_stats_flag_does_not_raise():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
     module.parse_arguments('--stats')
 
 
 def test_parse_arguments_allows_json_with_list_or_info():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
     module.parse_arguments('--list', '--json')
     module.parse_arguments('--info', '--json')
 
 
 def test_parse_arguments_disallows_json_without_list_or_info():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
     with pytest.raises(ValueError):
         module.parse_arguments('--json')
 
 
 def test_parse_arguments_disallows_json_with_both_list_and_info():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
     with pytest.raises(ValueError):
         module.parse_arguments('--list', '--info', '--json')
 
 
 def test_borgmatic_version_matches_news_version():
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+
     borgmatic_version = subprocess.check_output(('borgmatic', '--version')).decode('ascii')
     news_version = open('NEWS').readline()
 

tests/integration/config/test_load.py

@@ -0,0 +1,40 @@
+import sys
+
+from flexmock import flexmock
+
+from borgmatic.config import load as module
+
+
+def test_load_configuration_parses_contents():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('config.yaml').and_return('key: value')
+
+    assert module.load_configuration('config.yaml') == {'key': 'value'}
+
+
+def test_load_configuration_inlines_include():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return('value')
+    builtins.should_receive('open').with_args('config.yaml').and_return(
+        'key: !include include.yaml'
+    )
+
+    assert module.load_configuration('config.yaml') == {'key': 'value'}
+
+
+def test_load_configuration_merges_include():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return(
+        '''
+        foo: bar
+        baz: quux
+        '''
+    )
+    builtins.should_receive('open').with_args('config.yaml').and_return(
+        '''
+        foo: override
+        <<: !include include.yaml
+        '''
+    )
+
+    assert module.load_configuration('config.yaml') == {'foo': 'override', 'baz': 'quux'}

tests/integration/config/test_validate.py

@@ -118,6 +118,67 @@ def test_parse_configuration_with_schema_lacking_examples_does_not_raise():
     module.parse_configuration('config.yaml', 'schema.yaml')
 
 
+def test_parse_configuration_inlines_include():
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories:
+                - /home
+
+            repositories:
+                - hostname.borg
+
+        retention:
+            !include include.yaml
+        '''
+    )
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return(
+        '''
+        keep_daily: 7
+        keep_hourly: 24
+        '''
+    )
+
+    result = module.parse_configuration('config.yaml', 'schema.yaml')
+
+    assert result == {
+        'location': {'source_directories': ['/home'], 'repositories': ['hostname.borg']},
+        'retention': {'keep_daily': 7, 'keep_hourly': 24},
+    }
+
+
+def test_parse_configuration_merges_include():
+    mock_config_and_schema(
+        '''
+        location:
+            source_directories:
+                - /home
+
+            repositories:
+                - hostname.borg
+
+        retention:
+            keep_daily: 1
+            <<: !include include.yaml
+        '''
+    )
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('include.yaml').and_return(
+        '''
+        keep_daily: 7
+        keep_hourly: 24
+        '''
+    )
+
+    result = module.parse_configuration('config.yaml', 'schema.yaml')
+
+    assert result == {
+        'location': {'source_directories': ['/home'], 'repositories': ['hostname.borg']},
+        'retention': {'keep_daily': 1, 'keep_hourly': 24},
+    }
+
+
 def test_parse_configuration_raises_for_missing_config_file():
     with pytest.raises(FileNotFoundError):
         module.parse_configuration('config.yaml', 'schema.yaml')

tests/unit/borg/test_create.py

@@ -69,11 +69,6 @@ def test_write_pattern_file_with_empty_exclude_patterns_does_not_raise():
     module._write_pattern_file([])
 
 
-def insert_subprocess_mock(check_call_command, **kwargs):
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
-
-
 def test_make_pattern_flags_includes_pattern_filename_when_given():
     pattern_flags = module._make_pattern_flags(
         location_config={'patterns': ['R /', '- /var']}, pattern_filename='/tmp/patterns'
@@ -169,7 +164,9 @@ def test_create_archive_calls_borg_with_parameters():
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND)
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND, capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -192,7 +189,9 @@ def test_create_archive_with_patterns_calls_borg_with_patterns():
     ).and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(pattern_flags)
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + pattern_flags)
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + pattern_flags, capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -215,7 +214,9 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
     )
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(exclude_flags)
-    insert_subprocess_mock(CREATE_COMMAND + exclude_flags)
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + exclude_flags, capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -236,7 +237,9 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--list', '--filter', 'AME-', '--info', '--stats'))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--list', '--filter', 'AME-', '--info', '--stats'), capture_output=False
+    )
     insert_logging_mock(logging.INFO)
 
     module.create_archive(
@@ -257,8 +260,9 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(
-        CREATE_COMMAND + ('--list', '--filter', 'AME-', '--stats', '--debug', '--show-rc')
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--list', '--filter', 'AME-', '--stats', '--debug', '--show-rc'),
+        capture_output=False,
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -281,7 +285,9 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--dry-run',))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--dry-run',), capture_output=False
+    )
 
     module.create_archive(
         dry_run=True,
@@ -304,7 +310,9 @@ def test_create_archive_with_dry_run_and_log_info_calls_borg_without_stats_param
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--list', '--filter', 'AME-', '--info', '--dry-run'))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--list', '--filter', 'AME-', '--info', '--dry-run'), capture_output=False
+    )
     insert_logging_mock(logging.INFO)
 
     module.create_archive(
@@ -328,8 +336,9 @@ def test_create_archive_with_dry_run_and_log_debug_calls_borg_without_stats_para
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(
-        CREATE_COMMAND + ('--list', '--filter', 'AME-', '--debug', '--show-rc', '--dry-run')
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--list', '--filter', 'AME-', '--debug', '--show-rc', '--dry-run'),
+        capture_output=False,
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -351,7 +360,9 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--checkpoint-interval', '600'))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--checkpoint-interval', '600'), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -371,7 +382,9 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--chunker-params', '1,2,3,4'))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--chunker-params', '1,2,3,4'), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -391,7 +404,9 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle'))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--compression', 'rle'), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -411,7 +426,9 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--remote-ratelimit', '100'))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--remote-ratelimit', '100'), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -425,13 +442,15 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
     )
 
 
-def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameters():
+def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameter():
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--one-file-system',), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -446,13 +465,38 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
     )
 
 
+def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_parameter():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--numeric-owner',), capture_output=False
+    )
+
+    module.create_archive(
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'numeric_owner': True,
+            'exclude_patterns': None,
+        },
+        storage_config={},
+    )
+
+
 def test_create_archive_with_read_special_calls_borg_with_read_special_parameter():
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--read-special',))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--read-special',), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -473,7 +517,9 @@ def test_create_archive_with_bsd_flags_true_calls_borg_without_nobsdflags_parame
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND)
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND, capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -494,7 +540,9 @@ def test_create_archive_with_bsd_flags_false_calls_borg_with_nobsdflags_paramete
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--nobsdflags',))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--nobsdflags',), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -515,7 +563,9 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--files-cache', 'ctime,size'))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--files-cache', 'ctime,size'), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -536,7 +586,9 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(('borg1',) + CREATE_COMMAND[1:])
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg1',) + CREATE_COMMAND[1:], capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -557,7 +609,9 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--remote-path', 'borg1'))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--remote-path', 'borg1'), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -578,7 +632,9 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740'))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--umask', '740'), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -598,7 +654,9 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--lock-wait', '5'))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--lock-wait', '5'), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -618,9 +676,11 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(CREATE_COMMAND + ('--json',))
+    flexmock(module).should_receive('execute_command').with_args(
+        CREATE_COMMAND + ('--json',), capture_output=True
+    ).and_return('[]')
 
-    module.create_archive(
+    json_output = module.create_archive(
         dry_run=False,
         repository='repo',
         location_config={
@@ -632,6 +692,8 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
         json=True,
     )
 
+    assert json_output == '[]'
+
 
 def test_create_archive_with_source_directories_glob_expands():
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food'))
@@ -639,8 +701,9 @@ def test_create_archive_with_source_directories_glob_expands():
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(
-        ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food')
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'),
+        capture_output=False,
     )
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
 
@@ -662,7 +725,9 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo*'))
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo*'), capture_output=False
+    )
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([])
 
     module.create_archive(
@@ -683,8 +748,9 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(
-        ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food')
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'),
+        capture_output=False,
     )
 
     module.create_archive(
@@ -705,7 +771,9 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(('borg', 'create', 'repo::ARCHIVE_NAME', 'foo', 'bar'))
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', 'repo::ARCHIVE_NAME', 'foo', 'bar'), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,
@@ -725,7 +793,9 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    insert_subprocess_mock(('borg', 'create', 'repo::Documents_{hostname}-{now}', 'foo', 'bar'))
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', 'repo::Documents_{hostname}-{now}', 'foo', 'bar'), capture_output=False
+    )
 
     module.create_archive(
         dry_run=False,

tests/unit/borg/test_execute.py

@@ -0,0 +1,25 @@
+from flexmock import flexmock
+from borgmatic.borg import execute as module
+
+
+def test_execute_command_calls_full_command():
+    full_command = ['foo', 'bar']
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_call').with_args(full_command).once()
+
+    output = module.execute_command(full_command)
+
+    assert output is None
+
+
+def test_execute_command_captures_output():
+    full_command = ['foo', 'bar']
+    expected_output = '[]'
+    subprocess = flexmock(module.subprocess)
+    subprocess.should_receive('check_output').with_args(full_command).and_return(
+        flexmock(decode=lambda: expected_output)
+    ).once()
+
+    output = module.execute_command(full_command, capture_output=True)
+
+    assert output == expected_output

tests/unit/borg/test_extract.py

@@ -111,6 +111,7 @@ def test_extract_archive_calls_borg_with_restore_path_parameters():
         repository='repo',
         archive='archive',
         restore_paths=['path1', 'path2'],
+        location_config={},
         storage_config={},
     )
 
@@ -123,11 +124,25 @@ def test_extract_archive_calls_borg_with_remote_path_parameters():
         repository='repo',
         archive='archive',
         restore_paths=None,
+        location_config={},
         storage_config={},
         remote_path='borg1',
     )
 
 
+def test_extract_archive_calls_borg_with_numeric_owner_parameter():
+    insert_subprocess_mock(('borg', 'extract', 'repo::archive', '--numeric-owner'))
+
+    module.extract_archive(
+        dry_run=False,
+        repository='repo',
+        archive='archive',
+        restore_paths=None,
+        location_config={'numeric_owner': True},
+        storage_config={},
+    )
+
+
 def test_extract_archive_calls_borg_with_umask_parameters():
     insert_subprocess_mock(('borg', 'extract', 'repo::archive', '--umask', '0770'))
 
@@ -136,6 +151,7 @@ def test_extract_archive_calls_borg_with_umask_parameters():
         repository='repo',
         archive='archive',
         restore_paths=None,
+        location_config={},
         storage_config={'umask': '0770'},
     )
 
@@ -148,6 +164,7 @@ def test_extract_archive_calls_borg_with_lock_wait_parameters():
         repository='repo',
         archive='archive',
         restore_paths=None,
+        location_config={},
         storage_config={'lock_wait': '5'},
     )
 
@@ -157,7 +174,12 @@ def test_extract_archive_with_log_info_calls_borg_with_info_parameter():
     insert_logging_mock(logging.INFO)
 
     module.extract_archive(
-        dry_run=False, repository='repo', archive='archive', restore_paths=None, storage_config={}
+        dry_run=False,
+        repository='repo',
+        archive='archive',
+        restore_paths=None,
+        location_config={},
+        storage_config={},
     )
 
 
@@ -166,7 +188,12 @@ def test_extract_archive_with_log_debug_calls_borg_with_debug_parameters():
     insert_logging_mock(logging.DEBUG)
 
     module.extract_archive(
-        dry_run=False, repository='repo', archive='archive', restore_paths=None, storage_config={}
+        dry_run=False,
+        repository='repo',
+        archive='archive',
+        restore_paths=None,
+        location_config={},
+        storage_config={},
     )
 
 
@@ -174,7 +201,12 @@ def test_extract_archive_calls_borg_with_dry_run_parameter():
     insert_subprocess_mock(('borg', 'extract', 'repo::archive', '--dry-run'))
 
     module.extract_archive(
-        dry_run=True, repository='repo', archive='archive', restore_paths=None, storage_config={}
+        dry_run=True,
+        repository='repo',
+        archive='archive',
+        restore_paths=None,
+        location_config={},
+        storage_config={},
     )
 
 
@@ -186,6 +218,7 @@ def test_extract_archive_calls_borg_with_progress_parameter():
         repository='repo',
         archive='archive',
         restore_paths=None,
+        location_config={},
         storage_config={},
         progress=True,
     )

tests/unit/borg/test_info.py

@@ -6,53 +6,62 @@ from borgmatic.borg import info as module
 from ..test_verbosity import insert_logging_mock
 
 
-def insert_subprocess_mock(check_call_command, **kwargs):
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_output').with_args(check_call_command, **kwargs).once()
-
-
 INFO_COMMAND = ('borg', 'info', 'repo')
 
 
 def test_display_archives_info_calls_borg_with_parameters():
-    insert_subprocess_mock(INFO_COMMAND)
+    flexmock(module).should_receive('execute_command').with_args(INFO_COMMAND, capture_output=False)
 
     module.display_archives_info(repository='repo', storage_config={})
 
 
 def test_display_archives_info_with_log_info_calls_borg_with_info_parameter():
-    insert_subprocess_mock(INFO_COMMAND + ('--info',))
+    flexmock(module).should_receive('execute_command').with_args(
+        INFO_COMMAND + ('--info',), capture_output=False
+    )
     insert_logging_mock(logging.INFO)
     module.display_archives_info(repository='repo', storage_config={})
 
 
 def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter():
-    insert_subprocess_mock(INFO_COMMAND + ('--debug', '--show-rc'))
+    flexmock(module).should_receive('execute_command').with_args(
+        INFO_COMMAND + ('--debug', '--show-rc'), capture_output=False
+    )
     insert_logging_mock(logging.DEBUG)
 
     module.display_archives_info(repository='repo', storage_config={})
 
 
 def test_display_archives_info_with_json_calls_borg_with_json_parameter():
-    insert_subprocess_mock(INFO_COMMAND + ('--json',))
+    flexmock(module).should_receive('execute_command').with_args(
+        INFO_COMMAND + ('--json',), capture_output=True
+    ).and_return('[]')
 
-    module.display_archives_info(repository='repo', storage_config={}, json=True)
+    json_output = module.display_archives_info(repository='repo', storage_config={}, json=True)
+
+    assert json_output == '[]'
 
 
 def test_display_archives_info_with_local_path_calls_borg_via_local_path():
-    insert_subprocess_mock(('borg1',) + INFO_COMMAND[1:])
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg1',) + INFO_COMMAND[1:], capture_output=False
+    )
 
     module.display_archives_info(repository='repo', storage_config={}, local_path='borg1')
 
 
 def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_parameters():
-    insert_subprocess_mock(INFO_COMMAND + ('--remote-path', 'borg1'))
+    flexmock(module).should_receive('execute_command').with_args(
+        INFO_COMMAND + ('--remote-path', 'borg1'), capture_output=False
+    )
 
     module.display_archives_info(repository='repo', storage_config={}, remote_path='borg1')
 
 
 def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_parameters():
     storage_config = {'lock_wait': 5}
-    insert_subprocess_mock(INFO_COMMAND + ('--lock-wait', '5'))
+    flexmock(module).should_receive('execute_command').with_args(
+        INFO_COMMAND + ('--lock-wait', '5'), capture_output=False
+    )
 
     module.display_archives_info(repository='repo', storage_config=storage_config)

tests/unit/borg/test_list.py

@@ -6,54 +6,72 @@ from borgmatic.borg import list as module
 from ..test_verbosity import insert_logging_mock
 
 
-def insert_subprocess_mock(check_call_command, **kwargs):
-    subprocess = flexmock(module.subprocess)
-    subprocess.should_receive('check_output').with_args(check_call_command, **kwargs).once()
-
-
 LIST_COMMAND = ('borg', 'list', 'repo')
 
 
 def test_list_archives_calls_borg_with_parameters():
-    insert_subprocess_mock(LIST_COMMAND)
+    flexmock(module).should_receive('execute_command').with_args(LIST_COMMAND, capture_output=False)
 
     module.list_archives(repository='repo', storage_config={})
 
 
 def test_list_archives_with_log_info_calls_borg_with_info_parameter():
-    insert_subprocess_mock(LIST_COMMAND + ('--info',))
+    flexmock(module).should_receive('execute_command').with_args(
+        LIST_COMMAND + ('--info',), capture_output=False
+    )
     insert_logging_mock(logging.INFO)
 
     module.list_archives(repository='repo', storage_config={})
 
 
 def test_list_archives_with_log_debug_calls_borg_with_debug_parameter():
-    insert_subprocess_mock(LIST_COMMAND + ('--debug', '--show-rc'))
+    flexmock(module).should_receive('execute_command').with_args(
+        LIST_COMMAND + ('--debug', '--show-rc'), capture_output=False
+    )
     insert_logging_mock(logging.DEBUG)
 
     module.list_archives(repository='repo', storage_config={})
 
 
-def test_list_archives_with_json_calls_borg_with_json_parameter():
-    insert_subprocess_mock(LIST_COMMAND + ('--json',))
+def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
+    storage_config = {'lock_wait': 5}
+    flexmock(module).should_receive('execute_command').with_args(
+        LIST_COMMAND + ('--lock-wait', '5'), capture_output=False
+    )
 
-    module.list_archives(repository='repo', storage_config={}, json=True)
+    module.list_archives(repository='repo', storage_config=storage_config)
+
+
+def test_list_archives_with_archive_calls_borg_with_archive_parameter():
+    storage_config = {}
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'list', 'repo::archive'), capture_output=False
+    )
+
+    module.list_archives(repository='repo', storage_config=storage_config, archive='archive')
 
 
 def test_list_archives_with_local_path_calls_borg_via_local_path():
-    insert_subprocess_mock(('borg1',) + LIST_COMMAND[1:])
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg1',) + LIST_COMMAND[1:], capture_output=False
+    )
 
     module.list_archives(repository='repo', storage_config={}, local_path='borg1')
 
 
 def test_list_archives_with_remote_path_calls_borg_with_remote_path_parameters():
-    insert_subprocess_mock(LIST_COMMAND + ('--remote-path', 'borg1'))
+    flexmock(module).should_receive('execute_command').with_args(
+        LIST_COMMAND + ('--remote-path', 'borg1'), capture_output=False
+    )
 
     module.list_archives(repository='repo', storage_config={}, remote_path='borg1')
 
 
-def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
-    storage_config = {'lock_wait': 5}
-    insert_subprocess_mock(LIST_COMMAND + ('--lock-wait', '5'))
+def test_list_archives_with_json_calls_borg_with_json_parameter():
+    flexmock(module).should_receive('execute_command').with_args(
+        LIST_COMMAND + ('--json',), capture_output=True
+    ).and_return('[]')
 
-    module.list_archives(repository='repo', storage_config=storage_config)
+    json_output = module.list_archives(repository='repo', storage_config={}, json=True)
+
+    assert json_output == '[]'

tests/unit/commands/test_borgmatic.py

@@ -1,71 +1,27 @@
-import json
-import sys
-
 from flexmock import flexmock
 
 from borgmatic.commands import borgmatic as module
 
 
-def test_run_commands_handles_multiple_json_outputs_in_array():
-    (
-        flexmock(module)
-        .should_receive('_run_commands_on_repository')
-        .times(3)
-        .replace_with(
-            lambda args, consistency, json_results, local_path, location, remote_path, retention, storage, unexpanded_repository: json_results.append(
-                {"whatever": unexpanded_repository}
-            )
-        )
-    )
-
-    (
-        flexmock(sys.stdout)
-        .should_call("write")
-        .with_args(
-            json.dumps(
-                json.loads(
-                    '''
-                        [
-                            {"whatever": "fake_repo1"},
-                            {"whatever": "fake_repo2"},
-                            {"whatever": "fake_repo3"}
-                        ]
-                    '''
-                )
-            )
-        )
-    )
-
-    module._run_commands(
-        args=flexmock(json=True),
-        consistency=None,
-        local_path=None,
-        location={'repositories': ['fake_repo1', 'fake_repo2', 'fake_repo3']},
-        remote_path=None,
-        retention=None,
-        storage=None,
-    )
-
-
 def test_collect_configuration_run_summary_logs_info_for_success():
     flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
-    flexmock(module).should_receive('run_configuration')
-    args = flexmock(extract=False)
+    flexmock(module).should_receive('run_configuration').and_return([])
+    args = flexmock(extract=False, list=False)
 
     logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
 
-    assert any(log for log in logs if log.levelno == module.logging.INFO)
+    assert all(log for log in logs if log.levelno == module.logging.INFO)
 
 
 def test_collect_configuration_run_summary_logs_info_for_success_with_extract():
     flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
     flexmock(module.validate).should_receive('guard_configuration_contains_repository')
-    flexmock(module).should_receive('run_configuration')
-    args = flexmock(extract=True, repository='repo')
+    flexmock(module).should_receive('run_configuration').and_return([])
+    args = flexmock(extract=True, list=False, repository='repo')
 
     logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
 
-    assert any(log for log in logs if log.levelno == module.logging.INFO)
+    assert all(log for log in logs if log.levelno == module.logging.INFO)
 
 
 def test_collect_configuration_run_summary_logs_critical_for_extract_with_repository_error():
@@ -73,16 +29,38 @@ def test_collect_configuration_run_summary_logs_critical_for_extract_with_reposi
     flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise(
         ValueError
     )
-    args = flexmock(extract=True, repository='repo')
+    args = flexmock(extract=True, list=False, repository='repo')
 
     logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
 
     assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
 
 
+def test_collect_configuration_run_summary_logs_critical_for_list_with_archive_and_repository_error():
+    flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
+    flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise(
+        ValueError
+    )
+    args = flexmock(extract=False, list=True, repository='repo', archive='test')
+
+    logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
+
+    assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
+
+
+def test_collect_configuration_run_summary_logs_info_for_success_with_list():
+    flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
+    flexmock(module).should_receive('run_configuration').and_return([])
+    args = flexmock(extract=False, list=True, repository='repo', archive=None)
+
+    logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
+
+    assert all(log for log in logs if log.levelno == module.logging.INFO)
+
+
 def test_collect_configuration_run_summary_logs_critical_for_parse_error():
     flexmock(module.validate).should_receive('parse_configuration').and_raise(ValueError)
-    args = flexmock(extract=False)
+    args = flexmock(extract=False, list=False)
 
     logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
 
@@ -93,7 +71,7 @@ def test_collect_configuration_run_summary_logs_critical_for_run_error():
     flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
     flexmock(module.validate).should_receive('guard_configuration_contains_repository')
     flexmock(module).should_receive('run_configuration').and_raise(ValueError)
-    args = flexmock(extract=False)
+    args = flexmock(extract=False, list=False)
 
     logs = tuple(module.collect_configuration_run_summary_logs(('test.yaml',), args=args))
 
@@ -102,9 +80,22 @@ def test_collect_configuration_run_summary_logs_critical_for_run_error():
 
 def test_collect_configuration_run_summary_logs_critical_for_missing_configs():
     flexmock(module.validate).should_receive('parse_configuration').and_return({'test.yaml': {}})
-    flexmock(module).should_receive('run_configuration')
-    args = flexmock(config_paths=(), extract=False)
+    flexmock(module).should_receive('run_configuration').and_return([])
+    args = flexmock(config_paths=(), extract=False, list=False)
 
     logs = tuple(module.collect_configuration_run_summary_logs(config_filenames=(), args=args))
 
     assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
+
+
+def test_collect_configuration_run_summary_logs_outputs_merged_json_results():
+    flexmock(module.validate).should_receive('parse_configuration').and_return(
+        {'test.yaml': {}, 'test2.yaml': {}}
+    )
+    flexmock(module).should_receive('run_configuration').and_return(['foo', 'bar']).and_return(
+        ['baz']
+    )
+    flexmock(module.sys.stdout).should_receive('write').with_args('["foo", "bar", "baz"]').once()
+    args = flexmock(extract=False, list=False)
+
+    tuple(module.collect_configuration_run_summary_logs(('test.yaml', 'test2.yaml'), args=args))

tests/unit/config/test_collect.py

@@ -1,3 +1,5 @@
+import sys
+
 from flexmock import flexmock
 
 from borgmatic.config import collect as module
@@ -37,7 +39,10 @@ def test_collect_config_filenames_collects_files_from_given_directories_and_igno
     mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False)
     mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar').and_return(True)
     mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.yaml').and_return(False)
-    flexmock(module.os).should_receive('listdir').and_return(['foo.yaml', 'bar', 'baz.yaml'])
+    flexmock(module.os).should_receive('listdir')
+    flexmock(sys.modules['builtins']).should_receive('sorted').and_return(
+        ['foo.yaml', 'bar', 'baz.yaml']
+    )
 
     config_filenames = tuple(module.collect_config_filenames(config_paths))
 
@@ -56,7 +61,10 @@ def test_collect_config_filenames_collects_files_from_given_directories_and_igno
     mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/foo.yaml').and_return(False)
     mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/bar.yaml~').and_return(False)
     mock_path.should_receive('isdir').with_args('/etc/borgmatic.d/baz.txt').and_return(False)
-    flexmock(module.os).should_receive('listdir').and_return(['foo.yaml', 'bar.yaml~', 'baz.txt'])
+    flexmock(module.os).should_receive('listdir')
+    flexmock(sys.modules['builtins']).should_receive('sorted').and_return(
+        ['foo.yaml', 'bar.yaml~', 'baz.txt']
+    )
 
     config_filenames = tuple(module.collect_config_filenames(config_paths))