Add an additional end-to-end database test.

Updated the other unit tests, as I had to re-arrange argument order
Added an 'all' test for the postgres end-to-end test.

Ran black formatter on it all.

NEWS

@@ -1,3 +1,31 @@
+1.5.4
+ * #310: Fix legitimate database dump command errors (exit code 1) not being treated as errors by
+   borgmatic.
+ * For database dumps, replace the named pipe on every borgmatic run. This prevent hangs on stale
+   pipes left over from previous runs.
+ * Fix error handling to handle more edge cases when executing commands.
+
+1.5.3
+ * #258: Stream database dumps and restores directly to/from Borg without using any additional
+   filesystem space. This feature is automatic, and works even on restores from archives made with
+   previous versions of borgmatic.
+ * #293: Documentation on macOS launchd permissions issues with work-around for Full Disk Access.
+ * Remove "borgmatic restore --progress" flag, as it now conflicts with streaming database restores.
+
+1.5.2
+ * #301: Fix MySQL restore error on "all" database dump by excluding system tables.
+ * Fix PostgreSQL restore error on "all" database dump by using "psql" for the restore instead of
+   "pg_restore".
+
+1.5.1
+ * #289: Tired of looking up the latest successful archive name in order to pass it to borgmatic
+   actions? Me too. Now you can specify "--archive latest" to all actions that accept an archive
+   flag.
+ * #290: Fix the "--stats" and "--files" flags so that they yield output at verbosity 0.
+ * Reduce the default verbosity of borgmatic logs sent to Healthchecks monitoring hook. Now, it's
+   warnings and errors only. You can increase the verbosity via the "--monitoring-verbosity" flag.
+ * Add security policy documentation in SECURITY.md.
+
 1.5.0
  * #245: Monitor backups with PagerDuty hook integration. See the documentation for more
    information: https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook

README.md

@@ -119,14 +119,22 @@ If you'd like to chat with borgmatic developers or users, head on over to the
 href="https://webchat.freenode.net/?channels=borgmatic">web chat</a> or a
 native <a href="irc://chat.freenode.net:6697">IRC client</a>.
 
-Other questions or comments? Contact <mailto:witten@torsion.org>.
+Also see the [security
+policy](https://torsion.org/borgmatic/docs/security-policy/) for any security
+issues.
+
+Other questions or comments? Contact
+[witten@torsion.org](mailto:witten@torsion.org).
 
 
 ### Contributing
 
 borgmatic is hosted at <https://torsion.org/borgmatic> with [source code
-available](https://projects.torsion.org/witten/borgmatic). It's also mirrored
-on [GitHub](https://github.com/witten/borgmatic) for convenience.
+available](https://projects.torsion.org/witten/borgmatic), and is also
+mirrored on [GitHub](https://github.com/witten/borgmatic) for convenience.
+
+borgmatic is licensed under the GNU General Public License version 3 or any
+later version.
 
 If you'd like to contribute to borgmatic development, please feel free to
 submit a [Pull Request](https://projects.torsion.org/witten/borgmatic/pulls)

SECURITY.md

@@ -0,0 +1,19 @@
+---
+title: Security policy
+permalink: security-policy/index.html
+---
+
+## Supported versions
+
+While we want to hear about security vulnerabilities in all versions of
+borgmatic, security fixes will only be made to the most recently released
+version. It's not practical for our small volunteer effort to maintain
+multiple different release branches and put out separate security patches for
+each.
+
+## Reporting a vulnerability
+
+If you find a security vulnerability, please [file a
+ticket](https://torsion.org/borgmatic/#issues) or [send email
+directly](mailto:witten@torsion.org) as appropriate. You should expect to hear
+back within a few days at most, and generally sooner.

borgmatic/borg/check.py

@@ -1,7 +1,7 @@
 import logging
 
 from borgmatic.borg import extract
-from borgmatic.execute import execute_command, execute_command_without_capture
+from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 DEFAULT_CHECKS = ('repository', 'archives')
 DEFAULT_PREFIX = '{hostname}-'
@@ -134,9 +134,9 @@ def check_archives(
         # The Borg repair option trigger an interactive prompt, which won't work when output is
         # captured. And progress messes with the terminal directly.
         if repair or progress:
-            execute_command_without_capture(full_command, error_on_warnings=True)
+            execute_command(full_command, output_file=DO_NOT_CAPTURE)
         else:
-            execute_command(full_command, error_on_warnings=True)
+            execute_command(full_command)
 
     if 'extract' in checks:
         extract.extract_last_archive_dry_run(repository, lock_wait, local_path, remote_path)

borgmatic/borg/create.py

@@ -4,7 +4,7 @@ import logging
 import os
 import tempfile
 
-from borgmatic.execute import execute_command, execute_command_without_capture
+from borgmatic.execute import DO_NOT_CAPTURE, execute_command, execute_command_with_processes
 
 logger = logging.getLogger(__name__)
 
@@ -125,6 +125,9 @@ def borgmatic_source_directories(borgmatic_source_directory):
     )
 
 
+DEFAULT_ARCHIVE_NAME_FORMAT = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
+
+
 def create_archive(
     dry_run,
     repository,
@@ -136,10 +139,14 @@ def create_archive(
     stats=False,
     json=False,
     files=False,
+    stream_processes=None,
 ):
     '''
     Given vebosity/dry-run flags, a local or remote repository path, a location config dict, and a
     storage config dict, create a Borg archive and return Borg's JSON output (if any).
+
+    If a sequence of stream processes is given (instances of subprocess.Popen), then execute the
+    create command while also triggering the given processes to produce output.
     '''
     sources = _expand_directories(
         location_config['source_directories']
@@ -157,8 +164,7 @@ def create_archive(
     umask = storage_config.get('umask', None)
     lock_wait = storage_config.get('lock_wait', None)
     files_cache = location_config.get('files_cache')
-    default_archive_name_format = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
-    archive_name_format = storage_config.get('archive_name_format', default_archive_name_format)
+    archive_name_format = storage_config.get('archive_name_format', DEFAULT_ARCHIVE_NAME_FORMAT)
     extra_borg_options = storage_config.get('extra_borg_options', {}).get('create', '')
 
     full_command = (
@@ -174,7 +180,7 @@ def create_archive(
         + (('--noatime',) if location_config.get('atime') is False else ())
         + (('--noctime',) if location_config.get('ctime') is False else ())
         + (('--nobirthtime',) if location_config.get('birthtime') is False else ())
-        + (('--read-special',) if location_config.get('read_special') else ())
+        + (('--read-special',) if (location_config.get('read_special') or stream_processes) else ())
         + (('--nobsdflags',) if location_config.get('bsd_flags') is False else ())
         + (('--files-cache', files_cache) if files_cache else ())
         + (('--remote-path', remote_path) if remote_path else ())
@@ -196,12 +202,6 @@ def create_archive(
         + sources
     )
 
-    # The progress output isn't compatible with captured and logged output, as progress messes with
-    # the terminal directly.
-    if progress:
-        execute_command_without_capture(full_command, error_on_warnings=False)
-        return
-
     if json:
         output_log_level = None
     elif (stats or files) and logger.getEffectiveLevel() == logging.WARNING:
@@ -209,4 +209,17 @@ def create_archive(
     else:
         output_log_level = logging.INFO
 
-    return execute_command(full_command, output_log_level, error_on_warnings=False)
+    # The progress output isn't compatible with captured and logged output, as progress messes with
+    # the terminal directly.
+    output_file = DO_NOT_CAPTURE if progress else None
+
+    if stream_processes:
+        return execute_command_with_processes(
+            full_command,
+            stream_processes,
+            output_log_level,
+            output_file,
+            borg_local_path=local_path,
+        )
+
+    return execute_command(full_command, output_log_level, output_file, borg_local_path=local_path)

borgmatic/borg/extract.py

@@ -1,7 +1,8 @@
 import logging
 import os
+import subprocess
 
-from borgmatic.execute import execute_command, execute_command_without_capture
+from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 logger = logging.getLogger(__name__)
 
@@ -27,7 +28,9 @@ def extract_last_archive_dry_run(repository, lock_wait=None, local_path='borg',
         + (repository,)
     )
 
-    list_output = execute_command(full_list_command, output_log_level=None, error_on_warnings=False)
+    list_output = execute_command(
+        full_list_command, output_log_level=None, borg_local_path=local_path
+    )
 
     try:
         last_archive_name = list_output.strip().splitlines()[-1]
@@ -48,7 +51,7 @@ def extract_last_archive_dry_run(repository, lock_wait=None, local_path='borg',
         )
     )
 
-    execute_command(full_extract_command, working_directory=None, error_on_warnings=True)
+    execute_command(full_extract_command, working_directory=None)
 
 
 def extract_archive(
@@ -62,17 +65,23 @@ def extract_archive(
     remote_path=None,
     destination_path=None,
     progress=False,
-    error_on_warnings=True,
+    extract_to_stdout=False,
 ):
     '''
     Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to
     restore from the archive, location/storage configuration dicts, optional local and remote Borg
     paths, and an optional destination path to extract to, extract the archive into the current
     directory.
+
+    If extract to stdout is True, then start the extraction streaming to stdout, and return that
+    extract process as an instance of subprocess.Popen.
     '''
     umask = storage_config.get('umask', None)
     lock_wait = storage_config.get('lock_wait', None)
 
+    if progress and extract_to_stdout:
+        raise ValueError('progress and extract_to_stdout cannot both be set')
+
     full_command = (
         (local_path, 'extract')
         + (('--remote-path', remote_path) if remote_path else ())
@@ -83,6 +92,7 @@ def extract_archive(
         + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--dry-run',) if dry_run else ())
         + (('--progress',) if progress else ())
+        + (('--stdout',) if extract_to_stdout else ())
         + ('::'.join((repository if ':' in repository else os.path.abspath(repository), archive)),)
         + (tuple(paths) if paths else ())
     )
@@ -90,13 +100,19 @@ def extract_archive(
     # The progress output isn't compatible with captured and logged output, as progress messes with
     # the terminal directly.
     if progress:
-        execute_command_without_capture(
-            full_command, working_directory=destination_path, error_on_warnings=error_on_warnings
+        return execute_command(
+            full_command, output_file=DO_NOT_CAPTURE, working_directory=destination_path
         )
-        return
+        return None
 
-    # Error on warnings by default, as Borg only gives a warning if the restore paths don't exist in
-    # the archive!
-    execute_command(
-        full_command, working_directory=destination_path, error_on_warnings=error_on_warnings
-    )
+    if extract_to_stdout:
+        return execute_command(
+            full_command,
+            output_file=subprocess.PIPE,
+            working_directory=destination_path,
+            run_to_completion=False,
+        )
+
+    # Don't give Borg local path, so as to error on warnings, as Borg only gives a warning if the
+    # restore paths don't exist in the archive!
+    execute_command(full_command, working_directory=destination_path)

borgmatic/borg/info.py

@@ -41,5 +41,5 @@ def display_archives_info(
     return execute_command(
         full_command,
         output_log_level=None if info_arguments.json else logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path=local_path,
     )

borgmatic/borg/init.py

@@ -1,7 +1,7 @@
 import logging
 import subprocess
 
-from borgmatic.execute import execute_command, execute_command_without_capture
+from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 logger = logging.getLogger(__name__)
 
@@ -54,5 +54,5 @@ def initialize_repository(
         + (repository,)
     )
 
-    # Don't use execute_command() here because it doesn't support interactive prompts.
-    execute_command_without_capture(init_command, error_on_warnings=False)
+    # Do not capture output here, so as to support interactive prompts.
+    execute_command(init_command, output_file=DO_NOT_CAPTURE, borg_local_path=local_path)

borgmatic/borg/list.py

@@ -11,6 +11,42 @@ logger = logging.getLogger(__name__)
 BORG_EXCLUDE_CHECKPOINTS_GLOB = '*[0123456789]'
 
 
+def resolve_archive_name(repository, archive, storage_config, local_path='borg', remote_path=None):
+    '''
+    Given a local or remote repository path, an archive name, a storage config dict, a local Borg
+    path, and a remote Borg path, simply return the archive name. But if the archive name is
+    "latest", then instead introspect the repository for the latest successful (non-checkpoint)
+    archive, and return its name.
+
+    Raise ValueError if "latest" is given but there are no archives in the repository.
+    '''
+    if archive != "latest":
+        return archive
+
+    lock_wait = storage_config.get('lock_wait', None)
+
+    full_command = (
+        (local_path, 'list')
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + make_flags('remote-path', remote_path)
+        + make_flags('lock-wait', lock_wait)
+        + make_flags('glob-archives', BORG_EXCLUDE_CHECKPOINTS_GLOB)
+        + make_flags('last', 1)
+        + ('--short', repository)
+    )
+
+    output = execute_command(full_command, output_log_level=None, borg_local_path=local_path)
+    try:
+        latest_archive = output.strip().splitlines()[-1]
+    except IndexError:
+        raise ValueError('No archives found in the repository')
+
+    logger.debug('{}: Latest archive is {}'.format(repository, latest_archive))
+
+    return latest_archive
+
+
 def list_archives(repository, storage_config, list_arguments, local_path='borg', remote_path=None):
     '''
     Given a local or remote repository path, a storage config dict, and the arguments to the list
@@ -49,5 +85,5 @@ def list_archives(repository, storage_config, list_arguments, local_path='borg',
     return execute_command(
         full_command,
         output_log_level=None if list_arguments.json else logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path=local_path,
     )

borgmatic/borg/mount.py

@@ -1,6 +1,6 @@
 import logging
 
-from borgmatic.execute import execute_command, execute_command_without_capture
+from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 logger = logging.getLogger(__name__)
 
@@ -40,7 +40,7 @@ def mount_archive(
 
     # Don't capture the output when foreground mode is used so that ctrl-C can work properly.
     if foreground:
-        execute_command_without_capture(full_command, error_on_warnings=False)
+        execute_command(full_command, output_file=DO_NOT_CAPTURE, borg_local_path=local_path)
         return
 
-    execute_command(full_command, error_on_warnings=False)
+    execute_command(full_command, borg_local_path=local_path)

borgmatic/borg/prune.py

@@ -72,4 +72,4 @@ def prune_archives(
     else:
         output_log_level = logging.INFO
 
-    execute_command(full_command, output_log_level=output_log_level, error_on_warnings=False)
+    execute_command(full_command, output_log_level=output_log_level, borg_local_path=local_path)

borgmatic/borg/umount.py

@@ -17,4 +17,4 @@ def unmount_archive(mount_point, local_path='borg'):
         + (mount_point,)
     )
 
-    execute_command(full_command, error_on_warnings=True)
+    execute_command(full_command)

borgmatic/commands/arguments.py

@@ -163,7 +163,7 @@ def parse_arguments(*unparsed_arguments):
         '--monitoring-verbosity',
         type=int,
         choices=range(-1, 3),
-        default=1,
+        default=0,
         help='Log verbose progress to monitoring integrations that support logging (from only errors to very verbose: -1, 0, 1, or 2)',
     )
     global_group.add_argument(
@@ -323,7 +323,9 @@ def parse_arguments(*unparsed_arguments):
         '--repository',
         help='Path of repository to extract, defaults to the configured repository if there is only one',
     )
-    extract_group.add_argument('--archive', help='Name of archive to extract', required=True)
+    extract_group.add_argument(
+        '--archive', help='Name of archive to extract (or "latest")', required=True
+    )
     extract_group.add_argument(
         '--path',
         '--restore-path',
@@ -361,7 +363,7 @@ def parse_arguments(*unparsed_arguments):
         '--repository',
         help='Path of repository to use, defaults to the configured repository if there is only one',
     )
-    mount_group.add_argument('--archive', help='Name of archive to mount')
+    mount_group.add_argument('--archive', help='Name of archive to mount (or "latest")')
     mount_group.add_argument(
         '--mount-point',
         metavar='PATH',
@@ -415,7 +417,9 @@ def parse_arguments(*unparsed_arguments):
         '--repository',
         help='Path of repository to restore from, defaults to the configured repository if there is only one',
     )
-    restore_group.add_argument('--archive', help='Name of archive to restore from', required=True)
+    restore_group.add_argument(
+        '--archive', help='Name of archive to restore from (or "latest")', required=True
+    )
     restore_group.add_argument(
         '--database',
         metavar='NAME',
@@ -423,13 +427,6 @@ def parse_arguments(*unparsed_arguments):
         dest='databases',
         help='Names of databases to restore from archive, defaults to all databases. Note that any databases to restore must be defined in borgmatic\'s configuration',
     )
-    restore_group.add_argument(
-        '--progress',
-        dest='progress',
-        default=False,
-        action='store_true',
-        help='Display progress for each database dump file as it is extracted from archive',
-    )
     restore_group.add_argument(
         '-h', '--help', action='help', help='Show this help message and exit'
     )
@@ -446,7 +443,7 @@ def parse_arguments(*unparsed_arguments):
         '--repository',
         help='Path of repository to list, defaults to the configured repository if there is only one',
     )
-    list_group.add_argument('--archive', help='Name of archive to list')
+    list_group.add_argument('--archive', help='Name of archive to list (or "latest")')
     list_group.add_argument(
         '--path',
         metavar='PATH',
@@ -508,7 +505,7 @@ def parse_arguments(*unparsed_arguments):
         '--repository',
         help='Path of repository to show info for, defaults to the configured repository if there is only one',
     )
-    info_group.add_argument('--archive', help='Name of archive to show info for')
+    info_group.add_argument('--archive', help='Name of archive to show info for (or "latest")')
     info_group.add_argument(
         '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
     )

borgmatic/commands/borgmatic.py

@@ -1,4 +1,5 @@
 import collections
+import copy
 import json
 import logging
 import os
@@ -82,14 +83,6 @@ def run_configuration(config_filename, config, arguments):
                 'pre-backup',
                 global_arguments.dry_run,
             )
-            dispatch.call_hooks(
-                'dump_databases',
-                hooks,
-                config_filename,
-                dump.DATABASE_HOOK_NAMES,
-                location,
-                global_arguments.dry_run,
-            )
         if 'check' in arguments:
             command.execute_hook(
                 hooks.get('before_check'),
@@ -261,6 +254,16 @@ def run_actions(
         )
     if 'create' in arguments:
         logger.info('{}: Creating archive{}'.format(repository, dry_run_label))
+        active_dumps = dispatch.call_hooks(
+            'dump_databases',
+            hooks,
+            repository,
+            dump.DATABASE_HOOK_NAMES,
+            location,
+            global_arguments.dry_run,
+        )
+        stream_processes = [process for processes in active_dumps.values() for process in processes]
+
         json_output = borg_create.create_archive(
             global_arguments.dry_run,
             repository,
@@ -272,9 +275,11 @@ def run_actions(
             stats=arguments['create'].stats,
             json=arguments['create'].json,
             files=arguments['create'].files,
+            stream_processes=stream_processes,
         )
         if json_output:
             yield json.loads(json_output)
+
     if 'check' in arguments and checks.repository_enabled_for_checks(repository, consistency):
         logger.info('{}: Running consistency checks'.format(repository))
         borg_check.check_archives(
@@ -297,7 +302,9 @@ def run_actions(
             borg_extract.extract_archive(
                 global_arguments.dry_run,
                 repository,
-                arguments['extract'].archive,
+                borg_list.resolve_archive_name(
+                    repository, arguments['extract'].archive, storage, local_path, remote_path
+                ),
                 arguments['extract'].paths,
                 location,
                 storage,
@@ -319,7 +326,9 @@ def run_actions(
 
             borg_mount.mount_archive(
                 repository,
-                arguments['mount'].archive,
+                borg_list.resolve_archive_name(
+                    repository, arguments['mount'].archive, storage, local_path, remote_path
+                ),
                 arguments['mount'].mount_point,
                 arguments['mount'].paths,
                 arguments['mount'].foreground,
@@ -342,65 +351,80 @@ def run_actions(
             if 'all' in restore_names:
                 restore_names = []
 
-            # Extract dumps for the named databases from the archive.
-            dump_patterns = dispatch.call_hooks(
-                'make_database_dump_patterns',
-                hooks,
-                repository,
-                dump.DATABASE_HOOK_NAMES,
-                location,
-                restore_names,
+            archive_name = borg_list.resolve_archive_name(
+                repository, arguments['restore'].archive, storage, local_path, remote_path
             )
+            found_names = set()
 
-            borg_extract.extract_archive(
-                global_arguments.dry_run,
-                repository,
-                arguments['restore'].archive,
-                dump.convert_glob_patterns_to_borg_patterns(
-                    dump.flatten_dump_patterns(dump_patterns, restore_names)
-                ),
-                location,
-                storage,
-                local_path=local_path,
-                remote_path=remote_path,
-                destination_path='/',
-                progress=arguments['restore'].progress,
-                # We don't want glob patterns that don't match to error.
-                error_on_warnings=False,
-            )
+            for hook_name, per_hook_restore_databases in hooks.items():
+                if hook_name not in dump.DATABASE_HOOK_NAMES:
+                    continue
 
-            # Map the restore names or detected dumps to the corresponding database configurations.
-            restore_databases = dump.get_per_hook_database_configurations(
-                hooks, restore_names, dump_patterns
-            )
+                for restore_database in per_hook_restore_databases:
+                    database_name = restore_database['name']
+                    if restore_names and database_name not in restore_names:
+                        continue
+
+                    found_names.add(database_name)
+                    dump_pattern = dispatch.call_hooks(
+                        'make_database_dump_pattern',
+                        hooks,
+                        repository,
+                        dump.DATABASE_HOOK_NAMES,
+                        location,
+                        database_name,
+                    )[hook_name]
+
+                    # Kick off a single database extract to stdout.
+                    extract_process = borg_extract.extract_archive(
+                        dry_run=global_arguments.dry_run,
+                        repository=repository,
+                        archive=archive_name,
+                        paths=dump.convert_glob_patterns_to_borg_patterns([dump_pattern]),
+                        location_config=location,
+                        storage_config=storage,
+                        local_path=local_path,
+                        remote_path=remote_path,
+                        destination_path='/',
+                        extract_to_stdout=True,
+                    )
+
+                    # Run a single database restore, consuming the extract stdout.
+                    dispatch.call_hooks(
+                        'restore_database_dump',
+                        {hook_name: [restore_database]},
+                        repository,
+                        dump.DATABASE_HOOK_NAMES,
+                        location,
+                        global_arguments.dry_run,
+                        extract_process,
+                    )
+
+            if not restore_names and not found_names:
+                raise ValueError('No databases were found to restore')
+
+            missing_names = sorted(set(restore_names) - found_names)
+            if missing_names:
+                raise ValueError(
+                    'Cannot restore database(s) {} missing from borgmatic\'s configuration'.format(
+                        ', '.join(missing_names)
+                    )
+                )
 
-            # Finally, restore the databases and cleanup the dumps.
-            dispatch.call_hooks(
-                'restore_database_dumps',
-                restore_databases,
-                repository,
-                dump.DATABASE_HOOK_NAMES,
-                location,
-                global_arguments.dry_run,
-            )
-            dispatch.call_hooks(
-                'remove_database_dumps',
-                restore_databases,
-                repository,
-                dump.DATABASE_HOOK_NAMES,
-                location,
-                global_arguments.dry_run,
-            )
     if 'list' in arguments:
         if arguments['list'].repository is None or validate.repositories_match(
             repository, arguments['list'].repository
         ):
-            if not arguments['list'].json:
+            list_arguments = copy.copy(arguments['list'])
+            if not list_arguments.json:
                 logger.warning('{}: Listing archives'.format(repository))
+            list_arguments.archive = borg_list.resolve_archive_name(
+                repository, list_arguments.archive, storage, local_path, remote_path
+            )
             json_output = borg_list.list_archives(
                 repository,
                 storage,
-                list_arguments=arguments['list'],
+                list_arguments=list_arguments,
                 local_path=local_path,
                 remote_path=remote_path,
             )
@@ -410,12 +434,16 @@ def run_actions(
         if arguments['info'].repository is None or validate.repositories_match(
             repository, arguments['info'].repository
         ):
-            if not arguments['info'].json:
+            info_arguments = copy.copy(arguments['info'])
+            if not info_arguments.json:
                 logger.warning('{}: Displaying summary info for archives'.format(repository))
+            info_arguments.archive = borg_list.resolve_archive_name(
+                repository, info_arguments.archive, storage, local_path, remote_path
+            )
             json_output = borg_info.display_archives_info(
                 repository,
                 storage,
-                info_arguments=arguments['info'],
+                info_arguments=info_arguments,
                 local_path=local_path,
                 remote_path=remote_path,
             )

borgmatic/execute.py

@@ -1,5 +1,7 @@
+import collections
 import logging
 import os
+import select
 import subprocess
 
 logger = logging.getLogger(__name__)
@@ -9,52 +11,135 @@ ERROR_OUTPUT_MAX_LINE_COUNT = 25
 BORG_ERROR_EXIT_CODE = 2
 
 
-def exit_code_indicates_error(command, exit_code, error_on_warnings=True):
+def exit_code_indicates_error(process, exit_code, borg_local_path=None):
     '''
-    Return True if the given exit code from running the command corresponds to an error.
-    If error on warnings is False, then treat exit code 1 as a warning instead of an error.
+    Return True if the given exit code from running a command corresponds to an error. If a Borg
+    local path is given and matches the process' command, then treat exit code 1 as a warning
+    instead of an error.
     '''
-    if error_on_warnings:
-        return bool(exit_code != 0)
+    if exit_code is None:
+        return False
 
-    return bool(exit_code >= BORG_ERROR_EXIT_CODE)
+    command = process.args.split(' ') if isinstance(process.args, str) else process.args
+
+    if borg_local_path and command[0] == borg_local_path:
+        return bool(exit_code >= BORG_ERROR_EXIT_CODE)
+
+    return bool(exit_code != 0)
 
 
-def log_output(command, process, output_buffer, output_log_level, error_on_warnings):
+def command_for_process(process):
     '''
-    Given a command already executed, its process opened by subprocess.Popen(), and the process'
-    relevant output buffer (stderr or stdout), log its output with the requested log level.
-    Additionally, raise a CalledProcessException if the process exits with an error (or a warning,
-    if error on warnings is True).
+    Given a process as an instance of subprocess.Popen, return the command string that was used to
+    invoke it.
     '''
-    last_lines = []
+    return process.args if isinstance(process.args, str) else ' '.join(process.args)
 
-    while process.poll() is None:
-        line = output_buffer.readline().rstrip().decode()
-        if not line:
+
+def output_buffer_for_process(process, exclude_stdouts):
+    '''
+    Given a process as an instance of subprocess.Popen and a sequence of stdouts to exclude, return
+    either the process's stdout or stderr. The idea is that if stdout is excluded for a process, we
+    still have stderr to log.
+    '''
+    return process.stderr if process.stdout in exclude_stdouts else process.stdout
+
+
+def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
+    '''
+    Given a sequence of subprocess.Popen() instances for multiple processes, log the output for each
+    process with the requested log level. Additionally, raise a CalledProcessError if a process
+    exits with an error (or a warning for exit code 1, if that process matches the Borg local path).
+
+    For simplicity, it's assumed that the output buffer for each process is its stdout. But if any
+    stdouts are given to exclude, then for any matching processes, log from their stderr instead.
+
+    Note that stdout for a process can be None if output is intentionally not captured. In which
+    case it won't be logged.
+    '''
+    # Map from output buffer to sequence of last lines.
+    buffer_last_lines = collections.defaultdict(list)
+    output_buffers = [
+        output_buffer_for_process(process, exclude_stdouts)
+        for process in processes
+        if process.stdout or process.stderr
+    ]
+
+    # Log output for each process until they all exit.
+    while True:
+        if output_buffers:
+            (ready_buffers, _, _) = select.select(output_buffers, [], [])
+
+            for ready_buffer in ready_buffers:
+                line = ready_buffer.readline().rstrip().decode()
+                if not line:
+                    continue
+
+                # Keep the last few lines of output in case the process errors, and we need the output for
+                # the exception below.
+                last_lines = buffer_last_lines[ready_buffer]
+                last_lines.append(line)
+                if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT:
+                    last_lines.pop(0)
+
+                logger.log(output_log_level, line)
+
+        for process in processes:
+            exit_code = process.poll() if output_buffers else process.wait()
+
+            # If any process errors, then raise accordingly.
+            if exit_code_indicates_error(process, exit_code, borg_local_path):
+                # If an error occurs, include its output in the raised exception so that we don't
+                # inadvertently hide error output.
+                output_buffer = output_buffer_for_process(process, exclude_stdouts)
+
+                last_lines = buffer_last_lines[output_buffer] if output_buffer else []
+                if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT:
+                    last_lines.insert(0, '...')
+
+                # Something has gone wrong. So vent each process' output buffer to prevent it from
+                # hanging. And then kill the process.
+                for other_process in processes:
+                    if other_process.poll() is None:
+                        other_process.stdout.read(0)
+                        other_process.kill()
+
+                raise subprocess.CalledProcessError(
+                    exit_code, command_for_process(process), '\n'.join(last_lines)
+                )
+
+        if all(process.poll() is not None for process in processes):
+            break
+
+    # Consume any remaining output that we missed (if any).
+    for process in processes:
+        output_buffer = output_buffer_for_process(process, exclude_stdouts)
+
+        if not output_buffer:
             continue
 
-        # Keep the last few lines of output in case the command errors, and we need the output for
-        # the exception below.
-        last_lines.append(line)
-        if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT:
-            last_lines.pop(0)
+        remaining_output = output_buffer.read().rstrip().decode()
 
-        logger.log(output_log_level, line)
+        if remaining_output:  # pragma: no cover
+            logger.log(output_log_level, remaining_output)
 
-    remaining_output = output_buffer.read().rstrip().decode()
-    if remaining_output:  # pragma: no cover
-        logger.log(output_log_level, remaining_output)
 
-    exit_code = process.poll()
+def log_command(full_command, input_file, output_file):
+    '''
+    Log the given command (a sequence of command/argument strings), along with its input/output file
+    paths.
+    '''
+    logger.debug(
+        ' '.join(full_command)
+        + (' < {}'.format(getattr(input_file, 'name', '')) if input_file else '')
+        + (' > {}'.format(getattr(output_file, 'name', '')) if output_file else '')
+    )
 
-    if exit_code_indicates_error(command, exit_code, error_on_warnings):
-        # If an error occurs, include its output in the raised exception so that we don't
-        # inadvertently hide error output.
-        if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT:
-            last_lines.insert(0, '...')
 
-        raise subprocess.CalledProcessError(exit_code, ' '.join(command), '\n'.join(last_lines))
+# An sentinel passed as an output file to execute_command() to indicate that the command's output
+# should be allowed to flow through to stdout without being captured for logging. Useful for
+# commands with interactive prompts or those that mess directly with the console.
+DO_NOT_CAPTURE = object()
 
 
 def execute_command(
@@ -65,64 +150,108 @@ def execute_command(
     shell=False,
     extra_environment=None,
     working_directory=None,
-    error_on_warnings=True,
+    borg_local_path=None,
+    run_to_completion=True,
 ):
     '''
     Execute the given command (a sequence of command/argument strings) and log its output at the
-    given log level. If output log level is None, instead capture and return the output. If an
-    open output file object is given, then write stdout to the file and only log stderr (but only
-    if an output log level is set). If an open input file object is given, then read stdin from the
-    file. If shell is True, execute the command within a shell. If an extra environment dict is
-    given, then use it to augment the current environment, and pass the result into the command. If
-    a working directory is given, use that as the present working directory when running the
-    command. If error on warnings is False, then treat exit code 1 as a warning instead of an error.
+    given log level. If output log level is None, instead capture and return the output. (Implies
+    run_to_completion.) If an open output file object is given, then write stdout to the file and
+    only log stderr (but only if an output log level is set). If an open input file object is given,
+    then read stdin from the file. If shell is True, execute the command within a shell. If an extra
+    environment dict is given, then use it to augment the current environment, and pass the result
+    into the command. If a working directory is given, use that as the present working directory
+    when running the command. If a Borg local path is given, and the command matches it (regardless
+    of arguments), treat exit code 1 as a warning instead of an error. If run to completion is
+    False, then return the process for the command without executing it to completion.
 
     Raise subprocesses.CalledProcessError if an error occurs while running the command.
     '''
-    logger.debug(
-        ' '.join(full_command)
-        + (' < {}'.format(input_file.name) if input_file else '')
-        + (' > {}'.format(output_file.name) if output_file else '')
-    )
+    log_command(full_command, input_file, output_file)
     environment = {**os.environ, **extra_environment} if extra_environment else None
+    do_not_capture = bool(output_file is DO_NOT_CAPTURE)
+    command = ' '.join(full_command) if shell else full_command
 
     if output_log_level is None:
         output = subprocess.check_output(
-            full_command, shell=shell, env=environment, cwd=working_directory
+            command, shell=shell, env=environment, cwd=working_directory
         )
         return output.decode() if output is not None else None
-    else:
-        process = subprocess.Popen(
-            full_command,
+
+    process = subprocess.Popen(
+        command,
+        stdin=input_file,
+        stdout=None if do_not_capture else (output_file or subprocess.PIPE),
+        stderr=None if do_not_capture else (subprocess.PIPE if output_file else subprocess.STDOUT),
+        shell=shell,
+        env=environment,
+        cwd=working_directory,
+    )
+    if not run_to_completion:
+        return process
+
+    log_outputs(
+        (process,), (input_file, output_file), output_log_level, borg_local_path=borg_local_path
+    )
+
+
+def execute_command_with_processes(
+    full_command,
+    processes,
+    output_log_level=logging.INFO,
+    output_file=None,
+    input_file=None,
+    shell=False,
+    extra_environment=None,
+    working_directory=None,
+    borg_local_path=None,
+):
+    '''
+    Execute the given command (a sequence of command/argument strings) and log its output at the
+    given log level. Simultaneously, continue to poll one or more active processes so that they
+    run as well. This is useful, for instance, for processes that are streaming output to a named
+    pipe that the given command is consuming from.
+
+    If an open output file object is given, then write stdout to the file and only log stderr (but
+    only if an output log level is set). If an open input file object is given, then read stdin from
+    the file.  If shell is True, execute the command within a shell. If an extra environment dict is
+    given, then use it to augment the current environment, and pass the result into the command. If
+    a working directory is given, use that as the present working directory when running the
+    command. If a Borg local path is given, then for any matching command or process (regardless of
+    arguments), treat exit code 1 as a warning instead of an error.
+
+    Raise subprocesses.CalledProcessError if an error occurs while running the command or in the
+    upstream process.
+    '''
+    log_command(full_command, input_file, output_file)
+    environment = {**os.environ, **extra_environment} if extra_environment else None
+    do_not_capture = bool(output_file is DO_NOT_CAPTURE)
+    command = ' '.join(full_command) if shell else full_command
+
+    try:
+        command_process = subprocess.Popen(
+            command,
             stdin=input_file,
-            stdout=output_file or subprocess.PIPE,
-            stderr=subprocess.PIPE if output_file else subprocess.STDOUT,
+            stdout=None if do_not_capture else (output_file or subprocess.PIPE),
+            stderr=None
+            if do_not_capture
+            else (subprocess.PIPE if output_file else subprocess.STDOUT),
             shell=shell,
             env=environment,
             cwd=working_directory,
         )
-        log_output(
-            full_command,
-            process,
-            process.stderr if output_file else process.stdout,
-            output_log_level,
-            error_on_warnings,
-        )
+    except (subprocess.CalledProcessError, OSError):
+        # Something has gone wrong. So vent each process' output buffer to prevent it from hanging.
+        # And then kill the process.
+        for process in processes:
+            if process.poll() is None:
+                process.stdout.read(0)
+                process.kill()
+        raise
 
-
-def execute_command_without_capture(full_command, working_directory=None, error_on_warnings=True):
-    '''
-    Execute the given command (a sequence of command/argument strings), but don't capture or log its
-    output in any way. This is necessary for commands that monkey with the terminal (e.g. progress
-    display) or provide interactive prompts.
-
-    If a working directory is given, use that as the present working directory when running the
-    command. If error on warnings is False, then treat exit code 1 as a warning instead of an error.
-    '''
-    logger.debug(' '.join(full_command))
-
-    try:
-        subprocess.check_call(full_command, cwd=working_directory)
-    except subprocess.CalledProcessError as error:
-        if exit_code_indicates_error(full_command, error.returncode, error_on_warnings):
-            raise
+    log_outputs(
+        tuple(processes) + (command_process,),
+        (input_file, output_file),
+        output_log_level,
+        borg_local_path=borg_local_path,
+    )

borgmatic/hooks/dump.py

@@ -1,4 +1,3 @@
-import glob
 import logging
 import os
 import shutil
@@ -34,24 +33,15 @@ def make_database_dump_filename(dump_path, name, hostname=None):
     return os.path.join(os.path.expanduser(dump_path), hostname or 'localhost', name)
 
 
-def flatten_dump_patterns(dump_patterns, names):
+def create_named_pipe_for_dump(dump_path):
     '''
-    Given a dict from a database hook name to glob patterns matching the dumps for the named
-    databases, flatten out all the glob patterns into a single sequence, and return it.
-
-    Raise ValueError if there are no resulting glob patterns, which indicates that databases are not
-    configured in borgmatic's configuration.
+    Create a named pipe at the given dump path.
     '''
-    flattened = [pattern for patterns in dump_patterns.values() for pattern in patterns]
+    os.makedirs(os.path.dirname(dump_path), mode=0o700, exist_ok=True)
+    if os.path.exists(dump_path):
+        os.remove(dump_path)
 
-    if not flattened:
-        raise ValueError(
-            'Cannot restore database(s) {} missing from borgmatic\'s configuration'.format(
-                ', '.join(names) or '"all"'
-            )
-        )
-
-    return flattened
+    os.mkfifo(dump_path, mode=0o600)
 
 
 def remove_database_dumps(dump_path, databases, database_type_name, log_prefix, dry_run):
@@ -100,80 +90,3 @@ def convert_glob_patterns_to_borg_patterns(patterns):
     patterns like "sh:etc/*".
     '''
     return ['sh:{}'.format(pattern.lstrip(os.path.sep)) for pattern in patterns]
-
-
-def get_database_names_from_dumps(patterns):
-    '''
-    Given a sequence of database dump patterns, find the corresponding database dumps on disk and
-    return the database names from their filenames.
-    '''
-    return [os.path.basename(dump_path) for pattern in patterns for dump_path in glob.glob(pattern)]
-
-
-def get_database_configurations(databases, names):
-    '''
-    Given the full database configuration dicts as per the configuration schema, and a sequence of
-    database names, filter down and yield the configuration for just the named databases.
-    Additionally, if a database configuration is named "all", project out that configuration for
-    each named database.
-    '''
-    named_databases = {database['name']: database for database in databases}
-
-    for name in names:
-        database = named_databases.get(name)
-        if database:
-            yield database
-            continue
-
-        if 'all' in named_databases:
-            yield {**named_databases['all'], **{'name': name}}
-            continue
-
-
-def get_per_hook_database_configurations(hooks, names, dump_patterns):
-    '''
-    Given the hooks configuration dict as per the configuration schema, a sequence of database
-    names to restore, and a dict from database hook name to glob patterns for matching dumps,
-    filter down the configuration for just the named databases.
-
-    If there are no named databases given, then find the corresponding database dumps on disk and
-    use the database names from their filenames. Additionally, if a database configuration is named
-    "all", project out that configuration for each named database.
-
-    Return the results as a dict from database hook name to a sequence of database configuration
-    dicts for that database type.
-
-    Raise ValueError if one of the database names cannot be matched to a database in borgmatic's
-    database configuration.
-    '''
-    hook_databases = {
-        hook_name: list(
-            get_database_configurations(
-                hooks.get(hook_name),
-                names or get_database_names_from_dumps(dump_patterns[hook_name]),
-            )
-        )
-        for hook_name in DATABASE_HOOK_NAMES
-        if hook_name in hooks
-    }
-
-    if not names or 'all' in names:
-        if not any(hook_databases.values()):
-            raise ValueError(
-                'Cannot restore database "all", as there are no database dumps in the archive'
-            )
-
-        return hook_databases
-
-    found_names = {
-        database['name'] for databases in hook_databases.values() for database in databases
-    }
-    missing_names = sorted(set(names) - found_names)
-    if missing_names:
-        raise ValueError(
-            'Cannot restore database(s) {} missing from borgmatic\'s configuration'.format(
-                ', '.join(missing_names)
-            )
-        )
-
-    return hook_databases

borgmatic/hooks/mysql.py

@@ -1,7 +1,6 @@
 import logging
-import os
 
-from borgmatic.execute import execute_command
+from borgmatic.execute import execute_command, execute_command_with_processes
 from borgmatic.hooks import dump
 
 logger = logging.getLogger(__name__)
@@ -16,43 +15,103 @@ def make_dump_path(location_config):  # pragma: no cover
     )
 
 
+SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys')
+
+
+def database_names_to_dump(database, extra_environment, log_prefix, dry_run_label):
+    '''
+    Given a requested database name, return the corresponding sequence of database names to dump.
+    In the case of "all", query for the names of databases on the configured host and return them,
+    excluding any system databases that will cause problems during restore.
+    '''
+    requested_name = database['name']
+
+    if requested_name != 'all':
+        return (requested_name,)
+
+    show_command = (
+        ('mysql',)
+        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--port', str(database['port'])) if 'port' in database else ())
+        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--user', database['username']) if 'username' in database else ())
+        + ('--skip-column-names', '--batch')
+        + ('--execute', 'show schemas')
+    )
+    logger.debug(
+        '{}: Querying for "all" MySQL databases to dump{}'.format(log_prefix, dry_run_label)
+    )
+    show_output = execute_command(
+        show_command, output_log_level=None, extra_environment=extra_environment
+    )
+
+    return tuple(
+        show_name
+        for show_name in show_output.strip().splitlines()
+        if show_name not in SYSTEM_DATABASE_NAMES
+    )
+
+
 def dump_databases(databases, log_prefix, location_config, dry_run):
     '''
-    Dump the given MySQL/MariaDB databases to disk. The databases are supplied as a sequence of
-    dicts, one dict describing each database as per the configuration schema. Use the given log
+    Dump the given MySQL/MariaDB databases to a named pipe. The databases are supplied as a sequence
+    of dicts, one dict describing each database as per the configuration schema. Use the given log
     prefix in any log entries. Use the given location configuration dict to construct the
-    destination path. If this is a dry run, then don't actually dump anything.
+    destination path.
+
+    Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named
+    pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence.
     '''
     dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
+    processes = []
 
     logger.info('{}: Dumping MySQL databases{}'.format(log_prefix, dry_run_label))
 
     for database in databases:
-        name = database['name']
+        requested_name = database['name']
         dump_filename = dump.make_database_dump_filename(
-            make_dump_path(location_config), name, database.get('hostname')
+            make_dump_path(location_config), requested_name, database.get('hostname')
         )
-        command = (
-            ('mysqldump', '--add-drop-database')
+        extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
+        dump_command_names = database_names_to_dump(
+            database, extra_environment, log_prefix, dry_run_label
+        )
+
+        dump_command = (
+            ('mysqldump',)
+            + ('--add-drop-database',)
             + (('--host', database['hostname']) if 'hostname' in database else ())
             + (('--port', str(database['port'])) if 'port' in database else ())
             + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
             + (('--user', database['username']) if 'username' in database else ())
             + (tuple(database['options'].split(' ')) if 'options' in database else ())
-            + (('--all-databases',) if name == 'all' else ('--databases', name))
+            + ('--databases',)
+            + dump_command_names
+            # Use shell redirection rather than execute_command(output_file=open(...)) to prevent
+            # the open() call on a named pipe from hanging the main borgmatic process.
+            + ('>', dump_filename)
         )
-        extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
 
         logger.debug(
             '{}: Dumping MySQL database {} to {}{}'.format(
-                log_prefix, name, dump_filename, dry_run_label
+                log_prefix, requested_name, dump_filename, dry_run_label
             )
         )
-        if not dry_run:
-            os.makedirs(os.path.dirname(dump_filename), mode=0o700, exist_ok=True)
+        if dry_run:
+            continue
+
+        dump.create_named_pipe_for_dump(dump_filename)
+
+        processes.append(
             execute_command(
-                command, output_file=open(dump_filename, 'w'), extra_environment=extra_environment
+                dump_command,
+                shell=True,
+                extra_environment=extra_environment,
+                run_to_completion=False,
             )
+        )
+
+    return processes
 
 
 def remove_database_dumps(databases, log_prefix, location_config, dry_run):  # pragma: no cover
@@ -67,45 +126,51 @@ def remove_database_dumps(databases, log_prefix, location_config, dry_run):  # p
     )
 
 
-def make_database_dump_patterns(databases, log_prefix, location_config, names):
+def make_database_dump_pattern(
+    databases, log_prefix, location_config, name=None
+):  # pragma: no cover
     '''
     Given a sequence of configurations dicts, a prefix to log with, a location configuration dict,
-    and a sequence of database names to match, return the corresponding glob patterns to match the
-    database dumps in an archive. An empty sequence of names indicates that the patterns should
-    match all dumps.
+    and a database name to match, return the corresponding glob patterns to match the database dump
+    in an archive.
     '''
-    return [
-        dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
-        for name in (names or ['*'])
-    ]
+    return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
 
 
-def restore_database_dumps(databases, log_prefix, location_config, dry_run):
+def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
     '''
-    Restore the given MySQL/MariaDB databases from disk. The databases are supplied as a sequence of
-    dicts, one dict describing each database as per the configuration schema. Use the given log
-    prefix in any log entries. Use the given location configuration dict to construct the
-    destination path. If this is a dry run, then don't actually restore anything.
+    Restore the given MySQL/MariaDB database from an extract stream. The database is supplied as a
+    one-element sequence containing a dict describing the database, as per the configuration schema.
+    Use the given log prefix in any log entries. If this is a dry run, then don't actually restore
+    anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce
+    output to consume.
     '''
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
 
-    for database in databases:
-        dump_filename = dump.make_database_dump_filename(
-            make_dump_path(location_config), database['name'], database.get('hostname')
-        )
-        restore_command = (
-            ('mysql', '--batch')
-            + (('--host', database['hostname']) if 'hostname' in database else ())
-            + (('--port', str(database['port'])) if 'port' in database else ())
-            + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
-            + (('--user', database['username']) if 'username' in database else ())
-        )
-        extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
+    if len(database_config) != 1:
+        raise ValueError('The database configuration value is invalid')
 
-        logger.debug(
-            '{}: Restoring MySQL database {}{}'.format(log_prefix, database['name'], dry_run_label)
-        )
-        if not dry_run:
-            execute_command(
-                restore_command, input_file=open(dump_filename), extra_environment=extra_environment
-            )
+    database = database_config[0]
+    restore_command = (
+        ('mysql', '--batch', '--verbose')
+        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--port', str(database['port'])) if 'port' in database else ())
+        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--user', database['username']) if 'username' in database else ())
+    )
+    extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
+
+    logger.debug(
+        '{}: Restoring MySQL database {}{}'.format(log_prefix, database['name'], dry_run_label)
+    )
+    if dry_run:
+        return
+
+    execute_command_with_processes(
+        restore_command,
+        [extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment=extra_environment,
+        borg_local_path=location_config.get('local_path', 'borg'),
+    )

borgmatic/hooks/postgresql.py

@@ -1,7 +1,6 @@
 import logging
-import os
 
-from borgmatic.execute import execute_command
+from borgmatic.execute import execute_command, execute_command_with_processes
 from borgmatic.hooks import dump
 
 logger = logging.getLogger(__name__)
@@ -18,12 +17,16 @@ def make_dump_path(location_config):  # pragma: no cover
 
 def dump_databases(databases, log_prefix, location_config, dry_run):
     '''
-    Dump the given PostgreSQL databases to disk. The databases are supplied as a sequence of dicts,
-    one dict describing each database as per the configuration schema. Use the given log prefix in
-    any log entries. Use the given location configuration dict to construct the destination path. If
-    this is a dry run, then don't actually dump anything.
+    Dump the given PostgreSQL databases to a named pipe. The databases are supplied as a sequence of
+    dicts, one dict describing each database as per the configuration schema. Use the given log
+    prefix in any log entries. Use the given location configuration dict to construct the
+    destination path.
+
+    Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named
+    pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence.
     '''
     dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
+    processes = []
 
     logger.info('{}: Dumping PostgreSQL databases{}'.format(log_prefix, dry_run_label))
 
@@ -34,14 +37,21 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
         )
         all_databases = bool(name == 'all')
         command = (
-            ('pg_dumpall' if all_databases else 'pg_dump', '--no-password', '--clean')
-            + ('--file', dump_filename)
+            (
+                'pg_dumpall' if all_databases else 'pg_dump',
+                '--no-password',
+                '--clean',
+                '--if-exists',
+            )
             + (('--host', database['hostname']) if 'hostname' in database else ())
             + (('--port', str(database['port'])) if 'port' in database else ())
             + (('--username', database['username']) if 'username' in database else ())
             + (() if all_databases else ('--format', database.get('format', 'custom')))
             + (tuple(database['options'].split(' ')) if 'options' in database else ())
             + (() if all_databases else (name,))
+            # Use shell redirection rather than the --file flag to sidestep synchronization issues
+            # when pg_dump/pg_dumpall tries to write to a named pipe.
+            + ('>', dump_filename)
         )
         extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
 
@@ -50,9 +60,18 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
                 log_prefix, name, dump_filename, dry_run_label
             )
         )
-        if not dry_run:
-            os.makedirs(os.path.dirname(dump_filename), mode=0o700, exist_ok=True)
-            execute_command(command, extra_environment=extra_environment)
+        if dry_run:
+            continue
+
+        dump.create_named_pipe_for_dump(dump_filename)
+
+        processes.append(
+            execute_command(
+                command, shell=True, extra_environment=extra_environment, run_to_completion=False
+            )
+        )
+
+    return processes
 
 
 def remove_database_dumps(databases, log_prefix, location_config, dry_run):  # pragma: no cover
@@ -67,55 +86,65 @@ def remove_database_dumps(databases, log_prefix, location_config, dry_run):  # p
     )
 
 
-def make_database_dump_patterns(databases, log_prefix, location_config, names):
+def make_database_dump_pattern(
+    databases, log_prefix, location_config, name=None
+):  # pragma: no cover
     '''
     Given a sequence of configurations dicts, a prefix to log with, a location configuration dict,
-    and a sequence of database names to match, return the corresponding glob patterns to match the
-    database dumps in an archive. An empty sequence of names indicates that the patterns should
-    match all dumps.
+    and a database name to match, return the corresponding glob patterns to match the database dump
+    in an archive.
     '''
-    return [
-        dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
-        for name in (names or ['*'])
-    ]
+    return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
 
 
-def restore_database_dumps(databases, log_prefix, location_config, dry_run):
+def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
     '''
-    Restore the given PostgreSQL databases from disk. The databases are supplied as a sequence of
-    dicts, one dict describing each database as per the configuration schema. Use the given log
-    prefix in any log entries. Use the given location configuration dict to construct the
-    destination path. If this is a dry run, then don't actually restore anything.
+    Restore the given PostgreSQL database from an extract stream. The database is supplied as a
+    one-element sequence containing a dict describing the database, as per the configuration schema.
+    Use the given log prefix in any log entries. If this is a dry run, then don't actually restore
+    anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce
+    output to consume.
     '''
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
 
-    for database in databases:
-        dump_filename = dump.make_database_dump_filename(
-            make_dump_path(location_config), database['name'], database.get('hostname')
-        )
-        restore_command = (
-            ('pg_restore', '--no-password', '--clean', '--if-exists', '--exit-on-error')
-            + (('--host', database['hostname']) if 'hostname' in database else ())
-            + (('--port', str(database['port'])) if 'port' in database else ())
-            + (('--username', database['username']) if 'username' in database else ())
-            + ('--dbname', database['name'])
-            + (dump_filename,)
-        )
-        extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
-        analyze_command = (
-            ('psql', '--no-password', '--quiet')
-            + (('--host', database['hostname']) if 'hostname' in database else ())
-            + (('--port', str(database['port'])) if 'port' in database else ())
-            + (('--username', database['username']) if 'username' in database else ())
-            + ('--dbname', database['name'])
-            + ('--command', 'ANALYZE')
-        )
+    if len(database_config) != 1:
+        raise ValueError('The database configuration value is invalid')
 
-        logger.debug(
-            '{}: Restoring PostgreSQL database {}{}'.format(
-                log_prefix, database['name'], dry_run_label
-            )
+    database = database_config[0]
+    all_databases = bool(database['name'] == 'all')
+    analyze_command = (
+        ('psql', '--no-password', '--quiet')
+        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--port', str(database['port'])) if 'port' in database else ())
+        + (('--username', database['username']) if 'username' in database else ())
+        + (('--dbname', database['name']) if not all_databases else ())
+        + ('--command', 'ANALYZE')
+    )
+    restore_command = (
+        ('psql' if all_databases else 'pg_restore', '--no-password')
+        + (
+            ('--if-exists', '--exit-on-error', '--clean', '--dbname', database['name'])
+            if not all_databases
+            else ()
         )
-        if not dry_run:
-            execute_command(restore_command, extra_environment=extra_environment)
-            execute_command(analyze_command, extra_environment=extra_environment)
+        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--port', str(database['port'])) if 'port' in database else ())
+        + (('--username', database['username']) if 'username' in database else ())
+    )
+    extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
+
+    logger.debug(
+        '{}: Restoring PostgreSQL database {}{}'.format(log_prefix, database['name'], dry_run_label)
+    )
+    if dry_run:
+        return
+
+    execute_command_with_processes(
+        restore_command,
+        [extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment=extra_environment,
+        borg_local_path=location_config.get('local_path', 'borg'),
+    )
+    execute_command(analyze_command, extra_environment=extra_environment)

docs/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.7.4-alpine3.10 as borgmatic
+FROM python:3.8.1-alpine3.11 as borgmatic
 
 COPY . /app
 RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
@@ -7,7 +7,7 @@ RUN borgmatic --help > /command-line.txt \
            echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \
            && borgmatic "$action" --help >> /command-line.txt; done
 
-FROM node:12.10.0-alpine as html
+FROM node:13.7.0-alpine as html
 
 ARG ENVIRONMENT=production
 

docs/SECURITY.md

@@ -0,0 +1,19 @@
+---
+title: Security policy
+permalink: security-policy/index.html
+---
+
+## Supported versions
+
+While we want to hear about security vulnerabilities in all versions of
+borgmatic, security fixes will only be made to the most recently released
+version. It's not practical for our small volunteer effort to maintain
+multiple different release branches and put out separate security patches for
+each.
+
+## Reporting a vulnerability
+
+If you find a security vulnerability, please [file a
+ticket](https://torsion.org/borgmatic/#issues) or [send email
+directly](mailto:witten@torsion.org) as appropriate. You should expect to hear
+back within a few days at most, and generally sooner.

docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md

@@ -10,7 +10,7 @@ buddy's sometimes-online server for that extra level of redundancy.
 
 But if you run borgmatic and your hard drive isn't plugged in, or your buddy's
 server is offline, then you'll get an annoying error message and the overall
-borgmatic run will fail (even if individual repositories complete just fine).
+borgmatic run will fail (even if individual repositories still complete).
 
 So what if you want borgmatic to swallow the error of a missing drive
 or an offline server, and continue trucking along? That's where the concept of
@@ -96,7 +96,7 @@ There are some caveats you should be aware of with this feature.
    a test to make sure that individual source directories are mounted and
    available. Use your imagination!
  * The soft failure feature also works for `before_prune`, `after_prune`,
-   `before_check`, and `after_check` hooks. However it is not implemented for
+   `before_check`, and `after_check` hooks. But it is not implemented for
    `before_everything` or `after_everything`.
 
 ## Related documentation

docs/how-to/backup-your-databases.md

@@ -22,13 +22,13 @@ hooks:
         - name: posts
 ```
 
-Prior to each backup, borgmatic dumps each configured database to a file
-and includes it in the backup. After the backup completes, borgmatic removes
-the database dump files to recover disk space.
+As part of each backup, borgmatic streams a database dump for each configured
+database directly to Borg, so it's included in the backup without consuming
+additional disk space.
 
-borgmatic creates these temporary dump files in `~/.borgmatic` by default. To
-customize this path, set the `borgmatic_source_directory` option in the
-`location` section of borgmatic's configuration.
+To support this, borgmatic creates temporary named pipes in `~/.borgmatic` by
+default. To customize this path, set the `borgmatic_source_directory` option
+in the `location` section of borgmatic's configuration.
 
 Here's a more involved example that connects to remote databases:
 
@@ -112,6 +112,12 @@ borgmatic restore --archive host-2019-01-02T04:06:07.080910
 
 (No borgmatic `restore` action? Upgrade borgmatic!)
 
+With newer versions of borgmatic, you can simplify this to:
+
+```bash
+borgmatic restore --archive latest
+```
+
 The `--archive` value is the name of the archive to restore from. This
 restores all databases dumps that borgmatic originally backed up to that
 archive.

docs/how-to/deal-with-very-large-backups.md

@@ -7,7 +7,7 @@ Borg itself is great for efficiently de-duplicating data across successive
 backup archives, even when dealing with very large repositories. But you may
 find that while borgmatic's default mode of "prune, create, and check" works
 well on small repositories, it's not so great on larger ones. That's because
-running the default consistency checks takes a long time on large
+running the default pruning and consistency checks take a long time on large
 repositories.
 
 ### A la carte actions
@@ -27,9 +27,18 @@ borgmatic check
 
 You can run with only one of these actions provided, or you can mix and match
 any number of them in a single borgmatic run. This supports approaches like
-making backups with `create` on a frequent schedule, while only running
-expensive consistency checks with `check` on a much less frequent basis from
-a separate cron job.
+skipping certain actions while running others. For instance, this skips
+`prune` and only runs `create` and `check`:
+
+```bash
+borgmatic create check
+```
+
+Or, you can make backups with `create` on a frequent schedule (e.g. with
+`borgmatic create` called from one cron job), while only running expensive
+consistency checks with `check` on a much less frequent basis (e.g. with
+`borgmatic check` called from a separate cron job).
+
 
 ### Consistency check configuration
 
@@ -47,6 +56,15 @@ consistency:
         - repository
 ```
 
+Here are the available checks from fastest to slowest:
+
+ * `repository`: Checks the consistency of the repository itself.
+ * `archives`: Checks all of the archives in the repository.
+ * `extract`: Performs an extraction dry-run of the most recent archive.
+ * `data`: Verifies the data integrity of all archives contents, decrypting and decompressing all data (implies `archives` as well).
+
+See [Borg's check documentation](https://borgbackup.readthedocs.io/en/stable/usage/check.html) for more information.
+
 If that's still too slow, you can disable consistency checks entirely,
 either for a single repository or for all repositories.
 

docs/how-to/extract-a-backup.md

@@ -31,6 +31,12 @@ borgmatic extract --archive host-2019-01-02T04:06:07.080910
 (No borgmatic `extract` action? Try the old-style `--extract`, or upgrade
 borgmatic!)
 
+With newer versions of borgmatic, you can simplify this to:
+
+```bash
+borgmatic extract --archive latest
+```
+
 The `--archive` value is the name of the archive to extract. This extracts the
 entire contents of the archive to the current directory, so make sure you're
 in the right place before running the command.
@@ -106,6 +112,12 @@ Omit the `--archive` flag to mount all archives (lazy-loaded):
 borgmatic mount --mount-point /mnt
 ```
 
+Or use the "latest" value for the archive to mount the latest successful archive:
+
+```bash
+borgmatic mount --archive latest --mount-point /mnt
+```
+
 If you'd like to restrict the mounted filesystem to only particular paths from
 your archive, use the `--path` flag, similar to the `extract` action above.
 For instance:

docs/how-to/monitor-your-backups.md

@@ -203,12 +203,21 @@ or it doesn't hear from borgmatic for a certain period of time.
 
 ## PagerDuty hook
 
-[PagerDuty](https://cronhub.io/) provides incident monitoring and alerting,
-and borgmatic has built-in integration with it. Once you create a PagerDuty
-account and <a
+In case you're new here: [borgmatic](https://torsion.org/borgmatic/) is
+simple, configuration-driven backup software for servers and workstations,
+powered by [Borg Backup](https://www.borgbackup.org/).
+
+[PagerDuty](https://www.pagerduty.com/) provides incident monitoring and
+alerting. borgmatic has built-in integration that can notify you via PagerDuty
+as soon as a backup fails, so you can make sure your backups keep working.
+
+First, create a PagerDuty account and <a
 href="https://support.pagerduty.com/docs/services-and-integrations">service</a>
-on their site, all you need to do is configure borgmatic with the unique
-"Integration Key" for your service. Here's an example:
+on their site. On the service, add an integration and set the Integration Type
+to "borgmatic".
+
+Then, configure borgmatic with the unique "Integration Key" for your service.
+Here's an example:
 
 
 ```yaml
@@ -226,6 +235,9 @@ You can configure PagerDuty to notify you by a [variety of
 mechanisms](https://support.pagerduty.com/docs/notifications) when backups
 fail.
 
+If you have any issues with the integration, [please contact
+us](https://torsion.org/borgmatic/#support-and-contributing).
+
 
 ## Scripting borgmatic
 
@@ -261,6 +273,18 @@ multiple different hosts into a single repository, then you'll need to get
 fancier with your archive listing. See `borg list --help` for more flags.
 
 
+### Latest backups
+
+All borgmatic actions that accept an "--archive" flag allow you to specify an
+archive name of "latest". This lets you get the latest successful archive
+without having to first run "borgmatic list" manually, which can be handy in
+automated scripts. Here's an example:
+
+```bash
+borgmatic info --archive latest
+```
+
+
 ## Related documentation
 
  * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)

docs/how-to/set-up-backups.md

@@ -3,6 +3,11 @@ title: How to set up backups with borgmatic
 ---
 ## Installation
 
+Many users need to backup system files that require privileged access, so
+these instructions install and run borgmatic as root. If you don't need to
+backup such files, then you are welcome to install and run borgmatic as a
+non-root user.
+
 First, [install
 Borg](https://borgbackup.readthedocs.io/en/stable/installation.html), at least
 version 1.1.
@@ -39,6 +44,7 @@ borgmatic:
  * [Fedora official](https://bodhi.fedoraproject.org/updates/?search=borgmatic)
  * [Fedora unofficial](https://copr.fedorainfracloud.org/coprs/heffer/borgmatic/)
  * [Arch Linux](https://www.archlinux.org/packages/community/any/borgmatic/)
+ * [Alpine Linux](https://pkgs.alpinelinux.org/packages?name=borgmatic)
  * [OpenBSD](http://ports.su/sysutils/borgmatic)
  * [openSUSE](https://software.opensuse.org/package/borgmatic)
  * [stand-alone binary](https://github.com/cmarquardt/borgmatic-binary)
@@ -132,7 +138,7 @@ this step if you already have a Borg repository.) To create a repository, run
 a command like the following:
 
 ```bash
-borgmatic init --encryption repokey
+sudo borgmatic init --encryption repokey
 ```
 
 (No borgmatic `init` action? Try the old-style `--init` flag, or upgrade
@@ -164,7 +170,7 @@ good idea to test that borgmatic is working. So to run borgmatic and start a
 backup, you can invoke it like this:
 
 ```bash
-borgmatic --verbosity 1
+sudo borgmatic --verbosity 1
 ```
 
 By default, this will also prune any old backups as per the configured
@@ -215,6 +221,13 @@ sudo systemctl enable --now borgmatic.timer
 Feel free to modify the timer file based on how frequently you'd like
 borgmatic to run.
 
+### launchd in macOS
+
+If you run borgmatic in macOS with launchd, you may encounter permissions
+issues when reading files to backup. If that happens to you, you may be
+interested in an [unofficial work-around for Full Disk
+Access](https://projects.torsion.org/witten/borgmatic/issues/293).
+
 ## Colored output
 
 Borgmatic produces colored terminal output by default. It is disabled when a

scripts/release

@@ -23,9 +23,10 @@ git push github $version
 rm -fr dist
 python3 setup.py bdist_wheel
 python3 setup.py sdist
-gpg --detach-sign --armor dist/*
-twine upload -r pypi dist/borgmatic-*.tar.gz
-twine upload -r pypi dist/borgmatic-*-py3-none-any.whl
+gpg --detach-sign --armor dist/borgmatic-*.tar.gz
+gpg --detach-sign --armor dist/borgmatic-*-py3-none-any.whl
+twine upload -r pypi dist/borgmatic-*.tar.gz dist/borgmatic-*.tar.gz.asc
+twine upload -r pypi dist/borgmatic-*-py3-none-any.whl dist/borgmatic-*-py3-none-any.whl.asc
 
 # Set release changelogs on projects.torsion.org and GitHub.
 release_changelog="$(cat NEWS | sed '/^$/q' | grep -v '^\S')"

scripts/run-full-dev-tests

@@ -11,4 +11,4 @@
 set -e
 
 docker-compose --file tests/end-to-end/docker-compose.yaml up --force-recreate \
-    --abort-on-container-exit
+    --renew-anon-volumes --abort-on-container-exit

scripts/run-full-tests

@@ -10,8 +10,8 @@
 
 set -e
 
-python -m pip install --upgrade pip==19.3.1
-pip install tox==3.14.1
+python -m pip install --upgrade pip==20.0.2
+pip install tox==3.14.3
 export COVERAGE_FILE=/tmp/.coverage
 tox --workdir /tmp/.tox
 apk add --no-cache borgbackup postgresql-client mariadb-client

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.5.0'
+VERSION = '1.5.4'
 
 
 setup(

tests/end-to-end/docker-compose.yaml

@@ -1,17 +1,17 @@
 version: '3'
 services:
   postgresql:
-    image: postgres:11.6-alpine
+    image: postgres:12.2-alpine
     environment:
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
   mysql:
-    image: mariadb:10.4
+    image: mariadb:10.5
     environment:
       MYSQL_ROOT_PASSWORD: test
       MYSQL_DATABASE: test
   tests:
-    image: python:3.7-alpine3.10
+    image: python:3.8-alpine3.11
     volumes:
       - "../..:/app:ro"
     tmpfs:

tests/end-to-end/test_borgmatic.py

@@ -68,7 +68,7 @@ def test_borgmatic_command():
         extracted_config_path = os.path.join(extract_path, config_path)
         assert open(extracted_config_path).read() == open(config_path).read()
 
-        # Exercise the info flag.
+        # Exercise the info action.
         output = subprocess.check_output(
             'borgmatic --config {} info --json'.format(config_path).split(' ')
         ).decode(sys.stdout.encoding)

tests/end-to-end/test_database.py

@@ -5,6 +5,8 @@ import subprocess
 import sys
 import tempfile
 
+import pytest
+
 
 def write_configuration(config_path, repository_path, borgmatic_source_directory):
     '''
@@ -29,11 +31,19 @@ hooks:
           hostname: postgresql
           username: postgres
           password: test
+        - name: all
+          hostname: postgresql
+          username: postgres
+          password: test
     mysql_databases:
         - name: test
           hostname: mysql
           username: root
           password: test
+        - name: all
+          hostname: mysql
+          username: root
+          password: test
 '''.format(
         config_path, repository_path, borgmatic_source_directory
     )
@@ -59,7 +69,7 @@ def test_database_dump_and_restore():
             'borgmatic -v 2 --config {} init --encryption repokey'.format(config_path).split(' ')
         )
 
-        # Run borgmatic to generate a backup archive including a database dump
+        # Run borgmatic to generate a backup archive including a database dump.
         subprocess.check_call('borgmatic create --config {} -v 2'.format(config_path).split(' '))
 
         # Get the created archive name.
@@ -81,3 +91,38 @@ def test_database_dump_and_restore():
     finally:
         os.chdir(original_working_directory)
         shutil.rmtree(temporary_directory)
+
+
+def test_database_dump_with_error_causes_borgmatic_to_exit():
+    # Create a Borg repository.
+    temporary_directory = tempfile.mkdtemp()
+    repository_path = os.path.join(temporary_directory, 'test.borg')
+    borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic')
+
+    original_working_directory = os.getcwd()
+
+    try:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+        write_configuration(config_path, repository_path, borgmatic_source_directory)
+
+        subprocess.check_call(
+            'borgmatic -v 2 --config {} init --encryption repokey'.format(config_path).split(' ')
+        )
+
+        # Run borgmatic with a config override such that the database dump fails.
+        with pytest.raises(subprocess.CalledProcessError):
+            subprocess.check_call(
+                [
+                    'borgmatic',
+                    'create',
+                    '--config',
+                    config_path,
+                    '-v',
+                    '2',
+                    '--override',
+                    "hooks.postgresql_databases=[{'name': 'nope'}]",
+                ]
+            )
+    finally:
+        os.chdir(original_working_directory)
+        shutil.rmtree(temporary_directory)

tests/integration/commands/test_arguments.py

@@ -393,12 +393,6 @@ def test_parse_arguments_allows_progress_and_extract():
     module.parse_arguments('--progress', 'extract', '--archive', 'test', 'list')
 
 
-def test_parse_arguments_allows_progress_and_restore():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--progress', 'restore', '--archive', 'test', 'list')
-
-
 def test_parse_arguments_disallows_progress_without_create():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 

tests/integration/test_execute.py

@@ -7,74 +7,140 @@ from flexmock import flexmock
 from borgmatic import execute as module
 
 
-def test_log_output_logs_each_line_separately():
+def test_log_outputs_logs_each_line_separately():
     flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'hi').once()
     flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'there').once()
     flexmock(module).should_receive('exit_code_indicates_error').and_return(False)
 
     hi_process = subprocess.Popen(['echo', 'hi'], stdout=subprocess.PIPE)
-    module.log_output(
-        ['echo', 'hi'],
-        hi_process,
-        hi_process.stdout,
-        output_log_level=logging.INFO,
-        error_on_warnings=False,
-    )
+    flexmock(module).should_receive('output_buffer_for_process').with_args(
+        hi_process, ()
+    ).and_return(hi_process.stdout)
 
     there_process = subprocess.Popen(['echo', 'there'], stdout=subprocess.PIPE)
-    module.log_output(
-        ['echo', 'there'],
-        there_process,
-        there_process.stdout,
+    flexmock(module).should_receive('output_buffer_for_process').with_args(
+        there_process, ()
+    ).and_return(there_process.stdout)
+
+    module.log_outputs(
+        (hi_process, there_process),
+        exclude_stdouts=(),
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
 
-def test_log_output_includes_error_output_in_exception():
+def test_log_outputs_skips_logs_for_process_with_none_stdout():
+    flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'hi').never()
+    flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'there').once()
+    flexmock(module).should_receive('exit_code_indicates_error').and_return(False)
+
+    hi_process = subprocess.Popen(['echo', 'hi'], stdout=None)
+    flexmock(module).should_receive('output_buffer_for_process').with_args(
+        hi_process, ()
+    ).and_return(hi_process.stdout)
+
+    there_process = subprocess.Popen(['echo', 'there'], stdout=subprocess.PIPE)
+    flexmock(module).should_receive('output_buffer_for_process').with_args(
+        there_process, ()
+    ).and_return(there_process.stdout)
+
+    module.log_outputs(
+        (hi_process, there_process),
+        exclude_stdouts=(),
+        output_log_level=logging.INFO,
+        borg_local_path='borg',
+    )
+
+
+def test_log_outputs_includes_error_output_in_exception():
     flexmock(module.logger).should_receive('log')
     flexmock(module).should_receive('exit_code_indicates_error').and_return(True)
+    flexmock(module).should_receive('command_for_process').and_return('grep')
 
     process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout)
 
     with pytest.raises(subprocess.CalledProcessError) as error:
-        module.log_output(
-            ['grep'],
-            process,
-            process.stdout,
-            output_log_level=logging.INFO,
-            error_on_warnings=False,
+        module.log_outputs(
+            (process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
         )
 
     assert error.value.returncode == 2
     assert error.value.output
 
 
-def test_log_output_truncates_long_error_output():
+def test_log_outputs_skips_error_output_in_exception_for_process_with_none_stdout():
+    flexmock(module.logger).should_receive('log')
+    flexmock(module).should_receive('exit_code_indicates_error').and_return(True)
+    flexmock(module).should_receive('command_for_process').and_return('grep')
+
+    process = subprocess.Popen(['grep'], stdout=None)
+    flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout)
+
+    with pytest.raises(subprocess.CalledProcessError) as error:
+        module.log_outputs(
+            (process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
+        )
+
+    assert error.value.returncode == 2
+    assert not error.value.output
+
+
+def test_log_outputs_kills_other_processes_when_one_errors():
+    flexmock(module.logger).should_receive('log')
+    flexmock(module).should_receive('exit_code_indicates_error').and_return(True)
+    flexmock(module).should_receive('command_for_process').and_return('grep')
+
+    process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    other_process = subprocess.Popen(
+        ['watch', 'true'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+    )
+    flexmock(module).should_receive('output_buffer_for_process').with_args(process, ()).and_return(
+        process.stdout
+    )
+    flexmock(module).should_receive('output_buffer_for_process').with_args(
+        other_process, ()
+    ).and_return(other_process.stdout)
+    flexmock(other_process).should_receive('kill').once()
+
+    with pytest.raises(subprocess.CalledProcessError) as error:
+        module.log_outputs(
+            (process, other_process),
+            exclude_stdouts=(),
+            output_log_level=logging.INFO,
+            borg_local_path='borg',
+        )
+
+    assert error.value.returncode == 2
+    assert error.value.output
+
+
+def test_log_outputs_truncates_long_error_output():
     flexmock(module).ERROR_OUTPUT_MAX_LINE_COUNT = 0
     flexmock(module.logger).should_receive('log')
     flexmock(module).should_receive('exit_code_indicates_error').and_return(True)
+    flexmock(module).should_receive('command_for_process').and_return('grep')
 
     process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout)
 
     with pytest.raises(subprocess.CalledProcessError) as error:
-        module.log_output(
-            ['grep'],
-            process,
-            process.stdout,
-            output_log_level=logging.INFO,
-            error_on_warnings=False,
+        module.log_outputs(
+            (process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
         )
 
     assert error.value.returncode == 2
     assert error.value.output.startswith('...')
 
 
-def test_log_output_with_no_output_logs_nothing():
+def test_log_outputs_with_no_output_logs_nothing():
     flexmock(module.logger).should_receive('log').never()
     flexmock(module).should_receive('exit_code_indicates_error').and_return(False)
 
     process = subprocess.Popen(['true'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    module.log_output(
-        ['true'], process, process.stdout, output_log_level=logging.INFO, error_on_warnings=False
+    flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout)
+
+    module.log_outputs(
+        (process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
     )

tests/unit/borg/test_check.py

@@ -9,9 +9,7 @@ from ..test_verbosity import insert_logging_mock
 
 
 def insert_execute_command_mock(command):
-    flexmock(module).should_receive('execute_command').with_args(
-        command, error_on_warnings=True
-    ).once()
+    flexmock(module).should_receive('execute_command').with_args(command).once()
 
 
 def insert_execute_command_never():
@@ -164,8 +162,8 @@ def test_check_archives_with_progress_calls_borg_with_progress_parameter():
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').and_return(())
     flexmock(module).should_receive('execute_command').never()
-    flexmock(module).should_receive('execute_command_without_capture').with_args(
-        ('borg', 'check', '--progress', 'repo'), error_on_warnings=True
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'check', '--progress', 'repo'), output_file=module.DO_NOT_CAPTURE
     ).once()
 
     module.check_archives(
@@ -179,8 +177,8 @@ def test_check_archives_with_repair_calls_borg_with_repair_parameter():
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').and_return(())
     flexmock(module).should_receive('execute_command').never()
-    flexmock(module).should_receive('execute_command_without_capture').with_args(
-        ('borg', 'check', '--repair', 'repo'), error_on_warnings=True
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'check', '--repair', 'repo'), output_file=module.DO_NOT_CAPTURE
     ).once()
 
     module.check_archives(

tests/unit/borg/test_create.py

@@ -222,7 +222,8 @@ def test_create_archive_calls_borg_with_parameters():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -250,7 +251,8 @@ def test_create_archive_with_patterns_calls_borg_with_patterns():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create') + pattern_flags + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -278,7 +280,8 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create') + exclude_flags + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -304,7 +307,8 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--info') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.INFO)
 
@@ -331,7 +335,8 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--json') + ARCHIVE_WITH_PATHS,
         output_log_level=None,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.INFO)
 
@@ -358,7 +363,8 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--debug', '--show-rc') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -384,7 +390,8 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--json') + ARCHIVE_WITH_PATHS,
         output_log_level=None,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -412,7 +419,8 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--dry-run') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -440,7 +448,8 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--info', '--dry-run') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.INFO)
 
@@ -467,7 +476,8 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--checkpoint-interval', '600') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -492,7 +502,8 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--chunker-params', '1,2,3,4') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -517,7 +528,8 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--compression', 'rle') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -542,7 +554,8 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--remote-ratelimit', '100') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -567,7 +580,8 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--one-file-system') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -593,7 +607,8 @@ def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_paramet
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--numeric-owner') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -619,7 +634,8 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--read-special') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -646,7 +662,8 @@ def test_create_archive_with_option_true_calls_borg_without_corresponding_parame
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -673,7 +690,8 @@ def test_create_archive_with_option_false_calls_borg_with_corresponding_paramete
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--no' + option_name.replace('_', '')) + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -699,7 +717,8 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--files-cache', 'ctime,size') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -725,7 +744,8 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg1', 'create') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg1',
     )
 
     module.create_archive(
@@ -751,7 +771,8 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--remote-path', 'borg1') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -777,7 +798,8 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--umask', '740') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -802,7 +824,8 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--lock-wait', '5') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -827,7 +850,8 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_warning_o
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--stats') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -853,7 +877,8 @@ def test_create_archive_with_stats_and_log_info_calls_borg_with_stats_parameter_
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--info', '--stats') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.INFO)
 
@@ -880,7 +905,8 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_warning_ou
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--list', '--filter', 'AME-') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -906,7 +932,8 @@ def test_create_archive_with_files_and_log_info_calls_borg_with_list_parameter_a
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--list', '--filter', 'AME-', '--info') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.INFO)
 
@@ -930,8 +957,11 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    flexmock(module).should_receive('execute_command_without_capture').with_args(
-        ('borg', 'create', '--info', '--progress') + ARCHIVE_WITH_PATHS, error_on_warnings=False
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--info', '--progress') + ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
+        output_file=module.DO_NOT_CAPTURE,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.INFO)
 
@@ -955,8 +985,11 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter():
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    flexmock(module).should_receive('execute_command_without_capture').with_args(
-        ('borg', 'create', '--progress') + ARCHIVE_WITH_PATHS, error_on_warnings=False
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--progress') + ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
+        output_file=module.DO_NOT_CAPTURE,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -972,6 +1005,36 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter():
     )
 
 
+def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progress_parameter():
+    processes = flexmock()
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        ('borg', 'create', '--read-special', '--progress') + ARCHIVE_WITH_PATHS,
+        processes=processes,
+        output_log_level=logging.INFO,
+        output_file=module.DO_NOT_CAPTURE,
+        borg_local_path='borg',
+    )
+
+    module.create_archive(
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+        progress=True,
+        stream_processes=processes,
+    )
+
+
 def test_create_archive_with_json_calls_borg_with_json_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
@@ -982,7 +1045,8 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--json') + ARCHIVE_WITH_PATHS,
         output_log_level=None,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     ).and_return('[]')
 
     json_output = module.create_archive(
@@ -1010,7 +1074,8 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter()
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--json') + ARCHIVE_WITH_PATHS,
         output_log_level=None,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     ).and_return('[]')
 
     json_output = module.create_archive(
@@ -1039,7 +1104,8 @@ def test_create_archive_with_source_directories_glob_expands():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'),
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
 
@@ -1065,7 +1131,8 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo*'),
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([])
 
@@ -1091,7 +1158,8 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'),
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -1116,7 +1184,8 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', 'repo::ARCHIVE_NAME', 'foo', 'bar'),
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -1141,7 +1210,8 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', 'repo::Documents_{hostname}-{now}', 'foo', 'bar'),
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -1166,7 +1236,8 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--extra', '--options') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -1179,3 +1250,32 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
         },
         storage_config={'extra_borg_options': {'create': '--extra --options'}},
     )
+
+
+def test_create_archive_with_stream_processes_calls_borg_with_processes():
+    processes = flexmock()
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        ('borg', 'create', '--read-special') + ARCHIVE_WITH_PATHS,
+        processes=processes,
+        output_log_level=logging.INFO,
+        output_file=None,
+        borg_local_path='borg',
+    )
+
+    module.create_archive(
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+        stream_processes=processes,
+    )

tests/unit/borg/test_extract.py

@@ -1,5 +1,6 @@
 import logging
 
+import pytest
 from flexmock import flexmock
 
 from borgmatic.borg import extract as module
@@ -7,15 +8,15 @@ from borgmatic.borg import extract as module
 from ..test_verbosity import insert_logging_mock
 
 
-def insert_execute_command_mock(command, working_directory=None, error_on_warnings=True):
+def insert_execute_command_mock(command, working_directory=None):
     flexmock(module).should_receive('execute_command').with_args(
-        command, working_directory=working_directory, error_on_warnings=error_on_warnings
+        command, working_directory=working_directory
     ).once()
 
 
 def insert_execute_command_output_mock(command, result):
     flexmock(module).should_receive('execute_command').with_args(
-        command, output_log_level=None, error_on_warnings=False
+        command, output_log_level=None, borg_local_path=command[0]
     ).and_return(result).once()
 
 
@@ -221,10 +222,10 @@ def test_extract_archive_calls_borg_with_destination_path():
 
 def test_extract_archive_calls_borg_with_progress_parameter():
     flexmock(module.os.path).should_receive('abspath').and_return('repo')
-    flexmock(module).should_receive('execute_command_without_capture').with_args(
+    flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'extract', '--progress', 'repo::archive'),
+        output_file=module.DO_NOT_CAPTURE,
         working_directory=None,
-        error_on_warnings=True,
     ).once()
 
     module.extract_archive(
@@ -238,10 +239,50 @@ def test_extract_archive_calls_borg_with_progress_parameter():
     )
 
 
+def test_extract_archive_with_progress_and_extract_to_stdout_raises():
+    flexmock(module).should_receive('execute_command').never()
+
+    with pytest.raises(ValueError):
+        module.extract_archive(
+            dry_run=False,
+            repository='repo',
+            archive='archive',
+            paths=None,
+            location_config={},
+            storage_config={},
+            progress=True,
+            extract_to_stdout=True,
+        )
+
+
+def test_extract_archive_calls_borg_with_stdout_parameter_and_returns_process():
+    flexmock(module.os.path).should_receive('abspath').and_return('repo')
+    process = flexmock()
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'extract', '--stdout', 'repo::archive'),
+        output_file=module.subprocess.PIPE,
+        working_directory=None,
+        run_to_completion=False,
+    ).and_return(process).once()
+
+    assert (
+        module.extract_archive(
+            dry_run=False,
+            repository='repo',
+            archive='archive',
+            paths=None,
+            location_config={},
+            storage_config={},
+            extract_to_stdout=True,
+        )
+        == process
+    )
+
+
 def test_extract_archive_skips_abspath_for_remote_repository():
     flexmock(module.os.path).should_receive('abspath').never()
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'extract', 'server:repo::archive'), working_directory=None, error_on_warnings=True
+        ('borg', 'extract', 'server:repo::archive'), working_directory=None
     ).once()
 
     module.extract_archive(

tests/unit/borg/test_info.py

@@ -10,7 +10,7 @@ from ..test_verbosity import insert_logging_mock
 
 def test_display_archives_info_calls_borg_with_parameters():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', 'repo'), output_log_level=logging.WARNING, error_on_warnings=False
+        ('borg', 'info', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg'
     )
 
     module.display_archives_info(
@@ -20,9 +20,7 @@ def test_display_archives_info_calls_borg_with_parameters():
 
 def test_display_archives_info_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', '--info', 'repo'),
-        output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        ('borg', 'info', '--info', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg'
     )
     insert_logging_mock(logging.INFO)
     module.display_archives_info(
@@ -32,7 +30,7 @@ def test_display_archives_info_with_log_info_calls_borg_with_info_parameter():
 
 def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', '--json', 'repo'), output_log_level=None, error_on_warnings=False
+        ('borg', 'info', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
     ).and_return('[]')
 
     insert_logging_mock(logging.INFO)
@@ -47,7 +45,7 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'info', '--debug', '--show-rc', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -58,7 +56,7 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter():
 
 def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', '--json', 'repo'), output_log_level=None, error_on_warnings=False
+        ('borg', 'info', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
     ).and_return('[]')
 
     insert_logging_mock(logging.DEBUG)
@@ -71,7 +69,7 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp
 
 def test_display_archives_info_with_json_calls_borg_with_json_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', '--json', 'repo'), output_log_level=None, error_on_warnings=False
+        ('borg', 'info', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
     ).and_return('[]')
 
     json_output = module.display_archives_info(
@@ -83,7 +81,7 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter():
 
 def test_display_archives_info_with_archive_calls_borg_with_archive_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', 'repo::archive'), output_log_level=logging.WARNING, error_on_warnings=False
+        ('borg', 'info', 'repo::archive'), output_log_level=logging.WARNING, borg_local_path='borg'
     )
 
     module.display_archives_info(
@@ -93,7 +91,7 @@ def test_display_archives_info_with_archive_calls_borg_with_archive_parameter():
 
 def test_display_archives_info_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg1', 'info', 'repo'), output_log_level=logging.WARNING, error_on_warnings=False
+        ('borg1', 'info', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg1'
     )
 
     module.display_archives_info(
@@ -108,7 +106,7 @@ def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_para
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'info', '--remote-path', 'borg1', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.display_archives_info(
@@ -124,7 +122,7 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'info', '--lock-wait', '5', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.display_archives_info(
@@ -139,7 +137,7 @@ def test_display_archives_info_passes_through_arguments_to_borg(argument_name):
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'info', '--' + argument_name.replace('_', '-'), 'value', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.display_archives_info(

tests/unit/borg/test_init.py

@@ -23,8 +23,8 @@ def insert_info_command_not_found_mock():
 
 
 def insert_init_command_mock(init_command, **kwargs):
-    flexmock(module).should_receive('execute_command_without_capture').with_args(
-        init_command, error_on_warnings=False
+    flexmock(module).should_receive('execute_command').with_args(
+        init_command, output_file=module.DO_NOT_CAPTURE, borg_local_path=init_command[0]
     ).once()
 
 
@@ -37,7 +37,7 @@ def test_initialize_repository_calls_borg_with_parameters():
 
 def test_initialize_repository_raises_for_borg_init_error():
     insert_info_command_not_found_mock()
-    flexmock(module).should_receive('execute_command_without_capture').and_raise(
+    flexmock(module).should_receive('execute_command').and_raise(
         module.subprocess.CalledProcessError(2, 'borg init')
     )
 
@@ -48,8 +48,7 @@ def test_initialize_repository_raises_for_borg_init_error():
 
 
 def test_initialize_repository_skips_initialization_when_repository_already_exists():
-    insert_info_command_found_mock()
-    flexmock(module).should_receive('execute_command_without_capture').never()
+    flexmock(module).should_receive('execute_command').once()
 
     module.initialize_repository(repository='repo', storage_config={}, encryption_mode='repokey')
 

tests/unit/borg/test_list.py

@@ -7,10 +7,110 @@ from borgmatic.borg import list as module
 
 from ..test_verbosity import insert_logging_mock
 
+BORG_LIST_LATEST_ARGUMENTS = (
+    '--glob-archives',
+    module.BORG_EXCLUDE_CHECKPOINTS_GLOB,
+    '--last',
+    '1',
+    '--short',
+    'repo',
+)
+
+
+def test_resolve_archive_name_passes_through_non_latest_archive_name():
+    archive = 'myhost-2030-01-01T14:41:17.647620'
+
+    assert module.resolve_archive_name('repo', archive, storage_config={}) == archive
+
+
+def test_resolve_archive_name_calls_borg_with_parameters():
+    expected_archive = 'archive-name'
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, output_log_level=None, borg_local_path='borg'
+    ).and_return(expected_archive + '\n')
+
+    assert module.resolve_archive_name('repo', 'latest', storage_config={}) == expected_archive
+
+
+def test_resolve_archive_name_with_log_info_calls_borg_with_info_parameter():
+    expected_archive = 'archive-name'
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'list', '--info') + BORG_LIST_LATEST_ARGUMENTS,
+        output_log_level=None,
+        borg_local_path='borg',
+    ).and_return(expected_archive + '\n')
+    insert_logging_mock(logging.INFO)
+
+    assert module.resolve_archive_name('repo', 'latest', storage_config={}) == expected_archive
+
+
+def test_resolve_archive_name_with_log_debug_calls_borg_with_debug_parameter():
+    expected_archive = 'archive-name'
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'list', '--debug', '--show-rc') + BORG_LIST_LATEST_ARGUMENTS,
+        output_log_level=None,
+        borg_local_path='borg',
+    ).and_return(expected_archive + '\n')
+    insert_logging_mock(logging.DEBUG)
+
+    assert module.resolve_archive_name('repo', 'latest', storage_config={}) == expected_archive
+
+
+def test_resolve_archive_name_with_local_path_calls_borg_via_local_path():
+    expected_archive = 'archive-name'
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg1', 'list') + BORG_LIST_LATEST_ARGUMENTS,
+        output_log_level=None,
+        borg_local_path='borg1',
+    ).and_return(expected_archive + '\n')
+
+    assert (
+        module.resolve_archive_name('repo', 'latest', storage_config={}, local_path='borg1')
+        == expected_archive
+    )
+
+
+def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_parameters():
+    expected_archive = 'archive-name'
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'list', '--remote-path', 'borg1') + BORG_LIST_LATEST_ARGUMENTS,
+        output_log_level=None,
+        borg_local_path='borg',
+    ).and_return(expected_archive + '\n')
+
+    assert (
+        module.resolve_archive_name('repo', 'latest', storage_config={}, remote_path='borg1')
+        == expected_archive
+    )
+
+
+def test_resolve_archive_name_without_archives_raises():
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, output_log_level=None, borg_local_path='borg'
+    ).and_return('')
+
+    with pytest.raises(ValueError):
+        module.resolve_archive_name('repo', 'latest', storage_config={})
+
+
+def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_parameters():
+    expected_archive = 'archive-name'
+
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'list', '--lock-wait', 'okay') + BORG_LIST_LATEST_ARGUMENTS,
+        output_log_level=None,
+        borg_local_path='borg',
+    ).and_return(expected_archive + '\n')
+
+    assert (
+        module.resolve_archive_name('repo', 'latest', storage_config={'lock_wait': 'okay'})
+        == expected_archive
+    )
+
 
 def test_list_archives_calls_borg_with_parameters():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', 'repo'), output_log_level=logging.WARNING, error_on_warnings=False
+        ('borg', 'list', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg'
     )
 
     module.list_archives(
@@ -22,9 +122,7 @@ def test_list_archives_calls_borg_with_parameters():
 
 def test_list_archives_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', '--info', 'repo'),
-        output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        ('borg', 'list', '--info', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg'
     )
     insert_logging_mock(logging.INFO)
 
@@ -37,7 +135,7 @@ def test_list_archives_with_log_info_calls_borg_with_info_parameter():
 
 def test_list_archives_with_log_info_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', '--json', 'repo'), output_log_level=None, error_on_warnings=False
+        ('borg', 'list', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
     )
     insert_logging_mock(logging.INFO)
 
@@ -52,7 +150,7 @@ def test_list_archives_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--debug', '--show-rc', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -65,7 +163,7 @@ def test_list_archives_with_log_debug_calls_borg_with_debug_parameter():
 
 def test_list_archives_with_log_debug_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', '--json', 'repo'), output_log_level=None, error_on_warnings=False
+        ('borg', 'list', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -81,7 +179,7 @@ def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--lock-wait', '5', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.list_archives(
@@ -94,7 +192,7 @@ def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
 def test_list_archives_with_archive_calls_borg_with_archive_parameter():
     storage_config = {}
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', 'repo::archive'), output_log_level=logging.WARNING, error_on_warnings=False
+        ('borg', 'list', 'repo::archive'), output_log_level=logging.WARNING, borg_local_path='borg'
     )
 
     module.list_archives(
@@ -109,7 +207,7 @@ def test_list_archives_with_path_calls_borg_with_path_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', 'repo::archive', 'var/lib'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.list_archives(
@@ -121,7 +219,7 @@ def test_list_archives_with_path_calls_borg_with_path_parameter():
 
 def test_list_archives_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg1', 'list', 'repo'), output_log_level=logging.WARNING, error_on_warnings=False
+        ('borg1', 'list', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg1'
     )
 
     module.list_archives(
@@ -136,7 +234,7 @@ def test_list_archives_with_remote_path_calls_borg_with_remote_path_parameters()
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--remote-path', 'borg1', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.list_archives(
@@ -151,7 +249,7 @@ def test_list_archives_with_short_calls_borg_with_short_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--short', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     ).and_return('[]')
 
     module.list_archives(
@@ -179,7 +277,7 @@ def test_list_archives_passes_through_arguments_to_borg(argument_name):
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--' + argument_name.replace('_', '-'), 'value', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     ).and_return('[]')
 
     module.list_archives(
@@ -195,7 +293,7 @@ def test_list_archives_with_successful_calls_borg_to_exclude_checkpoints():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--glob-archives', module.BORG_EXCLUDE_CHECKPOINTS_GLOB, 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     ).and_return('[]')
 
     module.list_archives(
@@ -207,7 +305,7 @@ def test_list_archives_with_successful_calls_borg_to_exclude_checkpoints():
 
 def test_list_archives_with_json_calls_borg_with_json_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', '--json', 'repo'), output_log_level=None, error_on_warnings=False
+        ('borg', 'list', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
     ).and_return('[]')
 
     json_output = module.list_archives(

tests/unit/borg/test_mount.py

@@ -9,7 +9,7 @@ from ..test_verbosity import insert_logging_mock
 
 def insert_execute_command_mock(command):
     flexmock(module).should_receive('execute_command').with_args(
-        command, error_on_warnings=False
+        command, borg_local_path='borg'
     ).once()
 
 
@@ -117,8 +117,10 @@ def test_mount_archive_with_log_debug_calls_borg_with_debug_parameters():
 
 
 def test_mount_archive_calls_borg_with_foreground_parameter():
-    flexmock(module).should_receive('execute_command_without_capture').with_args(
-        ('borg', 'mount', '--foreground', 'repo::archive', '/mnt'), error_on_warnings=False
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'mount', '--foreground', 'repo::archive', '/mnt'),
+        output_file=module.DO_NOT_CAPTURE,
+        borg_local_path='borg',
     ).once()
 
     module.mount_archive(

tests/unit/borg/test_prune.py

@@ -10,7 +10,7 @@ from ..test_verbosity import insert_logging_mock
 
 def insert_execute_command_mock(prune_command, output_log_level):
     flexmock(module).should_receive('execute_command').with_args(
-        prune_command, output_log_level=output_log_level, error_on_warnings=False
+        prune_command, output_log_level=output_log_level, borg_local_path=prune_command[0]
     ).once()
 
 

tests/unit/borg/test_umount.py

@@ -8,9 +8,7 @@ from ..test_verbosity import insert_logging_mock
 
 
 def insert_execute_command_mock(command):
-    flexmock(module).should_receive('execute_command').with_args(
-        command, error_on_warnings=True
-    ).once()
+    flexmock(module).should_receive('execute_command').with_args(command).once()
 
 
 def test_unmount_archive_calls_borg_with_required_parameters():

tests/unit/hooks/test_dump.py

@@ -34,27 +34,13 @@ def test_make_database_dump_filename_with_invalid_name_raises():
         module.make_database_dump_filename('databases', 'invalid/name')
 
 
-def test_flatten_dump_patterns_produces_list_of_all_patterns():
-    dump_patterns = {'postgresql_databases': ['*/glob', 'glob/*'], 'mysql_databases': ['*/*/*']}
-    expected_patterns = sorted(
-        dump_patterns['postgresql_databases'] + dump_patterns['mysql_databases']
-    )
+def test_create_named_pipe_for_dump_does_not_raise():
+    flexmock(module.os).should_receive('makedirs')
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    flexmock(module.os).should_receive('remove')
+    flexmock(module.os).should_receive('mkfifo')
 
-    assert sorted(module.flatten_dump_patterns(dump_patterns, ('bob',))) == expected_patterns
-
-
-def test_flatten_dump_patterns_with_no_patterns_errors():
-    dump_patterns = {'postgresql_databases': [], 'mysql_databases': []}
-
-    with pytest.raises(ValueError):
-        assert module.flatten_dump_patterns(dump_patterns, ('bob',))
-
-
-def test_flatten_dump_patterns_with_no_hooks_errors():
-    dump_patterns = {}
-
-    with pytest.raises(ValueError):
-        assert module.flatten_dump_patterns(dump_patterns, ('bob',))
+    module.create_named_pipe_for_dump('/path/to/pipe')
 
 
 def test_remove_database_dumps_removes_dump_for_each_database():
@@ -107,93 +93,3 @@ def test_remove_database_dumps_without_databases_does_not_raise():
 
 def test_convert_glob_patterns_to_borg_patterns_removes_leading_slash():
     assert module.convert_glob_patterns_to_borg_patterns(('/etc/foo/bar',)) == ['sh:etc/foo/bar']
-
-
-def test_get_database_names_from_dumps_gets_names_from_filenames_matching_globs():
-    flexmock(module.glob).should_receive('glob').and_return(
-        ('databases/localhost/foo',)
-    ).and_return(('databases/localhost/bar',)).and_return(())
-
-    assert module.get_database_names_from_dumps(
-        ('databases/*/foo', 'databases/*/bar', 'databases/*/baz')
-    ) == ['foo', 'bar']
-
-
-def test_get_database_configurations_only_produces_named_databases():
-    databases = [
-        {'name': 'foo', 'hostname': 'example.org'},
-        {'name': 'bar', 'hostname': 'example.com'},
-        {'name': 'baz', 'hostname': 'example.org'},
-    ]
-
-    assert list(module.get_database_configurations(databases, ('foo', 'baz'))) == [
-        {'name': 'foo', 'hostname': 'example.org'},
-        {'name': 'baz', 'hostname': 'example.org'},
-    ]
-
-
-def test_get_database_configurations_matches_all_database():
-    databases = [
-        {'name': 'foo', 'hostname': 'example.org'},
-        {'name': 'all', 'hostname': 'example.com'},
-    ]
-
-    assert list(module.get_database_configurations(databases, ('foo', 'bar', 'baz'))) == [
-        {'name': 'foo', 'hostname': 'example.org'},
-        {'name': 'bar', 'hostname': 'example.com'},
-        {'name': 'baz', 'hostname': 'example.com'},
-    ]
-
-
-def test_get_per_hook_database_configurations_partitions_by_hook():
-    hooks = {'postgresql_databases': [flexmock()]}
-    names = ('foo', 'bar')
-    dump_patterns = flexmock()
-    expected_config = {'postgresql_databases': [{'name': 'foo'}, {'name': 'bar'}]}
-    flexmock(module).should_receive('get_database_configurations').with_args(
-        hooks['postgresql_databases'], names
-    ).and_return(expected_config['postgresql_databases'])
-
-    config = module.get_per_hook_database_configurations(hooks, names, dump_patterns)
-
-    assert config == expected_config
-
-
-def test_get_per_hook_database_configurations_defaults_to_detected_database_names():
-    hooks = {'postgresql_databases': [flexmock()]}
-    names = ()
-    detected_names = flexmock()
-    dump_patterns = {'postgresql_databases': [flexmock()]}
-    expected_config = {'postgresql_databases': [flexmock()]}
-    flexmock(module).should_receive('get_database_names_from_dumps').and_return(detected_names)
-    flexmock(module).should_receive('get_database_configurations').with_args(
-        hooks['postgresql_databases'], detected_names
-    ).and_return(expected_config['postgresql_databases'])
-
-    config = module.get_per_hook_database_configurations(hooks, names, dump_patterns)
-
-    assert config == expected_config
-
-
-def test_get_per_hook_database_configurations_with_unknown_database_name_raises():
-    hooks = {'postgresql_databases': [flexmock()]}
-    names = ('foo', 'bar')
-    dump_patterns = flexmock()
-    flexmock(module).should_receive('get_database_configurations').with_args(
-        hooks['postgresql_databases'], names
-    ).and_return([])
-
-    with pytest.raises(ValueError):
-        module.get_per_hook_database_configurations(hooks, names, dump_patterns)
-
-
-def test_get_per_hook_database_configurations_with_all_and_no_archive_dumps_raises():
-    hooks = {'postgresql_databases': [flexmock()]}
-    names = ('foo', 'all')
-    dump_patterns = flexmock()
-    flexmock(module).should_receive('get_database_configurations').with_args(
-        hooks['postgresql_databases'], names
-    ).and_return([])
-
-    with pytest.raises(ValueError):
-        module.get_per_hook_database_configurations(hooks, names, dump_patterns)

tests/unit/hooks/test_mysql.py

@@ -1,28 +1,68 @@
-import sys
+import logging
 
+import pytest
 from flexmock import flexmock
 
 from borgmatic.hooks import mysql as module
 
 
+def test_database_names_to_dump_passes_through_name():
+    extra_environment = flexmock()
+    log_prefix = ''
+    dry_run_label = ''
+
+    names = module.database_names_to_dump(
+        {'name': 'foo'}, extra_environment, log_prefix, dry_run_label
+    )
+
+    assert names == ('foo',)
+
+
+def test_database_names_to_dump_queries_mysql_for_database_names():
+    extra_environment = flexmock()
+    log_prefix = ''
+    dry_run_label = ''
+    flexmock(module).should_receive('execute_command').with_args(
+        ('mysql', '--skip-column-names', '--batch', '--execute', 'show schemas'),
+        output_log_level=None,
+        extra_environment=extra_environment,
+    ).and_return('foo\nbar\nmysql\n').once()
+
+    names = module.database_names_to_dump(
+        {'name': 'all'}, extra_environment, log_prefix, dry_run_label
+    )
+
+    assert names == ('foo', 'bar')
+
+
 def test_dump_databases_runs_mysqldump_for_each_database():
     databases = [{'name': 'foo'}, {'name': 'bar'}]
-    output_file = flexmock()
+    processes = [flexmock(), flexmock()]
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     ).and_return('databases/localhost/bar')
-    flexmock(module.os).should_receive('makedirs')
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(output_file)
+    flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
+        ('bar',)
+    )
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
 
-    for name in ('foo', 'bar'):
+    for name, process in zip(('foo', 'bar'), processes):
         flexmock(module).should_receive('execute_command').with_args(
-            ('mysqldump', '--add-drop-database', '--databases', name),
-            output_file=output_file,
+            (
+                'mysqldump',
+                '--add-drop-database',
+                '--databases',
+                name,
+                '>',
+                'databases/localhost/{}'.format(name),
+            ),
+            shell=True,
             extra_environment=None,
-        ).once()
+            run_to_completion=False,
+        ).and_return(process).once()
 
-    module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == processes
 
 
 def test_dump_databases_with_dry_run_skips_mysqldump():
@@ -31,7 +71,10 @@ def test_dump_databases_with_dry_run_skips_mysqldump():
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     ).and_return('databases/localhost/bar')
-    flexmock(module.os).should_receive('makedirs').never()
+    flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
+        ('bar',)
+    )
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
     flexmock(module).should_receive('execute_command').never()
 
     module.dump_databases(databases, 'test.yaml', {}, dry_run=True)
@@ -39,13 +82,13 @@ def test_dump_databases_with_dry_run_skips_mysqldump():
 
 def test_dump_databases_runs_mysqldump_with_hostname_and_port():
     databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
-    output_file = flexmock()
+    process = flexmock()
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/database.example.org/foo'
     )
-    flexmock(module.os).should_receive('makedirs')
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(output_file)
+    flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
 
     flexmock(module).should_receive('execute_command').with_args(
         (
@@ -59,128 +102,141 @@ def test_dump_databases_runs_mysqldump_with_hostname_and_port():
             'tcp',
             '--databases',
             'foo',
+            '>',
+            'databases/database.example.org/foo',
         ),
-        output_file=output_file,
+        shell=True,
         extra_environment=None,
-    ).once()
+        run_to_completion=False,
+    ).and_return(process).once()
 
-    module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
 
 
 def test_dump_databases_runs_mysqldump_with_username_and_password():
     databases = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}]
-    output_file = flexmock()
+    process = flexmock()
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     )
-    flexmock(module.os).should_receive('makedirs')
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(output_file)
+    flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
 
     flexmock(module).should_receive('execute_command').with_args(
-        ('mysqldump', '--add-drop-database', '--user', 'root', '--databases', 'foo'),
-        output_file=output_file,
+        (
+            'mysqldump',
+            '--add-drop-database',
+            '--user',
+            'root',
+            '--databases',
+            'foo',
+            '>',
+            'databases/localhost/foo',
+        ),
+        shell=True,
         extra_environment={'MYSQL_PWD': 'trustsome1'},
-    ).once()
+        run_to_completion=False,
+    ).and_return(process).once()
 
-    module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
 
 
 def test_dump_databases_runs_mysqldump_with_options():
     databases = [{'name': 'foo', 'options': '--stuff=such'}]
-    output_file = flexmock()
+    process = flexmock()
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     )
-    flexmock(module.os).should_receive('makedirs')
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(output_file)
+    flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
 
     flexmock(module).should_receive('execute_command').with_args(
-        ('mysqldump', '--add-drop-database', '--stuff=such', '--databases', 'foo'),
-        output_file=output_file,
+        (
+            'mysqldump',
+            '--add-drop-database',
+            '--stuff=such',
+            '--databases',
+            'foo',
+            '>',
+            'databases/localhost/foo',
+        ),
+        shell=True,
         extra_environment=None,
-    ).once()
+        run_to_completion=False,
+    ).and_return(process).once()
 
-    module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
 
 
 def test_dump_databases_runs_mysqldump_for_all_databases():
     databases = [{'name': 'all'}]
-    output_file = flexmock()
+    process = flexmock()
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/all'
     )
-    flexmock(module.os).should_receive('makedirs')
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(output_file)
-
-    flexmock(module).should_receive('execute_command').with_args(
-        ('mysqldump', '--add-drop-database', '--all-databases'),
-        output_file=output_file,
-        extra_environment=None,
-    ).once()
-
-    module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
-
-
-def test_make_database_dump_patterns_converts_names_to_glob_paths():
-    flexmock(module).should_receive('make_dump_path').and_return('')
-    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
-        'databases/*/foo'
-    ).and_return('databases/*/bar')
-
-    assert module.make_database_dump_patterns(flexmock(), flexmock(), {}, ('foo', 'bar')) == [
-        'databases/*/foo',
-        'databases/*/bar',
-    ]
-
-
-def test_make_database_dump_patterns_treats_empty_names_as_matching_all_databases():
-    flexmock(module).should_receive('make_dump_path').and_return('/dump/path')
-    flexmock(module.dump).should_receive('make_database_dump_filename').with_args(
-        '/dump/path', '*', '*'
-    ).and_return('databases/*/*')
-
-    assert module.make_database_dump_patterns(flexmock(), flexmock(), {}, ()) == ['databases/*/*']
-
-
-def test_restore_database_dumps_restores_each_database():
-    databases = [{'name': 'foo'}, {'name': 'bar'}]
-    flexmock(module).should_receive('make_dump_path').and_return('')
-    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
-        'databases/localhost/foo'
-    ).and_return('databases/localhost/bar')
-
-    for name in ('foo', 'bar'):
-        dump_filename = 'databases/localhost/{}'.format(name)
-        input_file = flexmock()
-        flexmock(sys.modules['builtins']).should_receive('open').with_args(
-            dump_filename
-        ).and_return(input_file)
-        flexmock(module).should_receive('execute_command').with_args(
-            ('mysql', '--batch'), input_file=input_file, extra_environment=None
-        ).once()
-
-    module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)
-
-
-def test_restore_database_dumps_runs_mysql_with_hostname_and_port():
-    databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
-    flexmock(module).should_receive('make_dump_path').and_return('')
-    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
-        'databases/localhost/foo'
-    )
-    dump_filename = 'databases/localhost/foo'
-    input_file = flexmock()
-    flexmock(sys.modules['builtins']).should_receive('open').with_args(dump_filename).and_return(
-        input_file
-    )
+    flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar'))
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
 
     flexmock(module).should_receive('execute_command').with_args(
+        (
+            'mysqldump',
+            '--add-drop-database',
+            '--databases',
+            'foo',
+            'bar',
+            '>',
+            'databases/localhost/all',
+        ),
+        shell=True,
+        extra_environment=None,
+        run_to_completion=False,
+    ).and_return(process).once()
+
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
+
+
+def test_restore_database_dump_runs_mysql_to_restore():
+    database_config = [{'name': 'foo'}]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        ('mysql', '--batch', '--verbose'),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment=None,
+        borg_local_path='borg',
+    ).once()
+
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+    )
+
+
+def test_restore_database_dump_errors_on_multiple_database_config():
+    database_config = [{'name': 'foo'}, {'name': 'bar'}]
+
+    flexmock(module).should_receive('execute_command_with_processes').never()
+    flexmock(module).should_receive('execute_command').never()
+
+    with pytest.raises(ValueError):
+        module.restore_database_dump(
+            database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock()
+        )
+
+
+def test_restore_database_dump_runs_mysql_with_hostname_and_port():
+    database_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
         (
             'mysql',
             '--batch',
+            '--verbose',
             '--host',
             'database.example.org',
             '--port',
@@ -188,29 +244,41 @@ def test_restore_database_dumps_runs_mysql_with_hostname_and_port():
             '--protocol',
             'tcp',
         ),
-        input_file=input_file,
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
         extra_environment=None,
+        borg_local_path='borg',
     ).once()
 
-    module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)
-
-
-def test_restore_database_dumps_runs_mysql_with_username_and_password():
-    databases = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}]
-    flexmock(module).should_receive('make_dump_path').and_return('')
-    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
-        'databases/localhost/foo'
-    )
-    dump_filename = 'databases/localhost/foo'
-    input_file = flexmock()
-    flexmock(sys.modules['builtins']).should_receive('open').with_args(dump_filename).and_return(
-        input_file
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
     )
 
-    flexmock(module).should_receive('execute_command').with_args(
-        ('mysql', '--batch', '--user', 'root'),
-        input_file=input_file,
+
+def test_restore_database_dump_runs_mysql_with_username_and_password():
+    database_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        ('mysql', '--batch', '--verbose', '--user', 'root'),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
         extra_environment={'MYSQL_PWD': 'trustsome1'},
+        borg_local_path='borg',
     ).once()
 
-    module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+    )
+
+
+def test_restore_database_dump_with_dry_run_skips_restore():
+    database_config = [{'name': 'foo'}]
+
+    flexmock(module).should_receive('execute_command_with_processes').never()
+
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock()
+    )

tests/unit/hooks/test_postgresql.py

@@ -1,3 +1,6 @@
+import logging
+
+import pytest
 from flexmock import flexmock
 
 from borgmatic.hooks import postgresql as module
@@ -5,28 +8,32 @@ from borgmatic.hooks import postgresql as module
 
 def test_dump_databases_runs_pg_dump_for_each_database():
     databases = [{'name': 'foo'}, {'name': 'bar'}]
+    processes = [flexmock(), flexmock()]
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     ).and_return('databases/localhost/bar')
-    flexmock(module.os).should_receive('makedirs')
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
 
-    for name in ('foo', 'bar'):
+    for name, process in zip(('foo', 'bar'), processes):
         flexmock(module).should_receive('execute_command').with_args(
             (
                 'pg_dump',
                 '--no-password',
                 '--clean',
-                '--file',
-                'databases/localhost/{}'.format(name),
+                '--if-exists',
                 '--format',
                 'custom',
                 name,
+                '>',
+                'databases/localhost/{}'.format(name),
             ),
+            shell=True,
             extra_environment=None,
-        ).once()
+            run_to_completion=False,
+        ).and_return(process).once()
 
-    module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == processes
 
 
 def test_dump_databases_with_dry_run_skips_pg_dump():
@@ -35,27 +42,27 @@ def test_dump_databases_with_dry_run_skips_pg_dump():
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     ).and_return('databases/localhost/bar')
-    flexmock(module.os).should_receive('makedirs').never()
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
     flexmock(module).should_receive('execute_command').never()
 
-    module.dump_databases(databases, 'test.yaml', {}, dry_run=True)
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=True) == []
 
 
 def test_dump_databases_runs_pg_dump_with_hostname_and_port():
     databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
+    process = flexmock()
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/database.example.org/foo'
     )
-    flexmock(module.os).should_receive('makedirs')
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
 
     flexmock(module).should_receive('execute_command').with_args(
         (
             'pg_dump',
             '--no-password',
             '--clean',
-            '--file',
-            'databases/database.example.org/foo',
+            '--if-exists',
             '--host',
             'database.example.org',
             '--port',
@@ -63,180 +70,191 @@ def test_dump_databases_runs_pg_dump_with_hostname_and_port():
             '--format',
             'custom',
             'foo',
+            '>',
+            'databases/database.example.org/foo',
         ),
+        shell=True,
         extra_environment=None,
-    ).once()
+        run_to_completion=False,
+    ).and_return(process).once()
 
-    module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
 
 
 def test_dump_databases_runs_pg_dump_with_username_and_password():
     databases = [{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1'}]
+    process = flexmock()
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     )
-    flexmock(module.os).should_receive('makedirs')
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
 
     flexmock(module).should_receive('execute_command').with_args(
         (
             'pg_dump',
             '--no-password',
             '--clean',
-            '--file',
-            'databases/localhost/foo',
+            '--if-exists',
             '--username',
             'postgres',
             '--format',
             'custom',
             'foo',
+            '>',
+            'databases/localhost/foo',
         ),
+        shell=True,
         extra_environment={'PGPASSWORD': 'trustsome1'},
-    ).once()
+        run_to_completion=False,
+    ).and_return(process).once()
 
-    module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
 
 
 def test_dump_databases_runs_pg_dump_with_format():
     databases = [{'name': 'foo', 'format': 'tar'}]
+    process = flexmock()
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     )
-    flexmock(module.os).should_receive('makedirs')
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
 
     flexmock(module).should_receive('execute_command').with_args(
         (
             'pg_dump',
             '--no-password',
             '--clean',
-            '--file',
-            'databases/localhost/foo',
+            '--if-exists',
             '--format',
             'tar',
             'foo',
+            '>',
+            'databases/localhost/foo',
         ),
+        shell=True,
         extra_environment=None,
-    ).once()
+        run_to_completion=False,
+    ).and_return(process).once()
 
-    module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
 
 
 def test_dump_databases_runs_pg_dump_with_options():
     databases = [{'name': 'foo', 'options': '--stuff=such'}]
+    process = flexmock()
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     )
-    flexmock(module.os).should_receive('makedirs')
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
 
     flexmock(module).should_receive('execute_command').with_args(
         (
             'pg_dump',
             '--no-password',
             '--clean',
-            '--file',
-            'databases/localhost/foo',
+            '--if-exists',
             '--format',
             'custom',
             '--stuff=such',
             'foo',
+            '>',
+            'databases/localhost/foo',
         ),
+        shell=True,
         extra_environment=None,
-    ).once()
+        run_to_completion=False,
+    ).and_return(process).once()
 
-    module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
 
 
 def test_dump_databases_runs_pg_dumpall_for_all_databases():
     databases = [{'name': 'all'}]
+    process = flexmock()
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/all'
     )
-    flexmock(module.os).should_receive('makedirs')
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
 
     flexmock(module).should_receive('execute_command').with_args(
-        ('pg_dumpall', '--no-password', '--clean', '--file', 'databases/localhost/all'),
+        ('pg_dumpall', '--no-password', '--clean', '--if-exists', '>', 'databases/localhost/all'),
+        shell=True,
         extra_environment=None,
-    ).once()
+        run_to_completion=False,
+    ).and_return(process).once()
 
-    module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
 
 
-def test_make_database_dump_patterns_converts_names_to_glob_paths():
-    flexmock(module).should_receive('make_dump_path').and_return('')
-    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
-        'databases/*/foo'
-    ).and_return('databases/*/bar')
+def test_restore_database_dump_runs_pg_restore():
+    database_config = [{'name': 'foo'}]
+    extract_process = flexmock(stdout=flexmock())
 
-    assert module.make_database_dump_patterns(flexmock(), flexmock(), {}, ('foo', 'bar')) == [
-        'databases/*/foo',
-        'databases/*/bar',
-    ]
-
-
-def test_make_database_dump_patterns_treats_empty_names_as_matching_all_databases():
-    flexmock(module).should_receive('make_dump_path').and_return('/dump/path')
-    flexmock(module.dump).should_receive('make_database_dump_filename').with_args(
-        '/dump/path', '*', '*'
-    ).and_return('databases/*/*')
-
-    assert module.make_database_dump_patterns(flexmock(), flexmock(), {}, ()) == ['databases/*/*']
-
-
-def test_restore_database_dumps_restores_each_database():
-    databases = [{'name': 'foo'}, {'name': 'bar'}]
-    flexmock(module).should_receive('make_dump_path').and_return('')
-    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
-        'databases/localhost/foo'
-    ).and_return('databases/localhost/bar')
-
-    for name in ('foo', 'bar'):
-        flexmock(module).should_receive('execute_command').with_args(
-            (
-                'pg_restore',
-                '--no-password',
-                '--clean',
-                '--if-exists',
-                '--exit-on-error',
-                '--dbname',
-                name,
-                'databases/localhost/{}'.format(name),
-            ),
-            extra_environment=None,
-        ).once()
-        flexmock(module).should_receive('execute_command').with_args(
-            ('psql', '--no-password', '--quiet', '--dbname', name, '--command', 'ANALYZE'),
-            extra_environment=None,
-        ).once()
-
-    module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)
-
-
-def test_restore_database_dumps_runs_pg_restore_with_hostname_and_port():
-    databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
-    flexmock(module).should_receive('make_dump_path').and_return('')
-    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
-        'databases/localhost/foo'
-    )
-
-    flexmock(module).should_receive('execute_command').with_args(
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
         (
             'pg_restore',
             '--no-password',
-            '--clean',
             '--if-exists',
             '--exit-on-error',
+            '--clean',
+            '--dbname',
+            'foo',
+        ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment=None,
+        borg_local_path='borg',
+    ).once()
+    flexmock(module).should_receive('execute_command').with_args(
+        ('psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'),
+        extra_environment=None,
+    ).once()
+
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+    )
+
+
+def test_restore_database_dump_errors_on_multiple_database_config():
+    database_config = [{'name': 'foo'}, {'name': 'bar'}]
+
+    flexmock(module).should_receive('execute_command_with_processes').never()
+    flexmock(module).should_receive('execute_command').never()
+
+    with pytest.raises(ValueError):
+        module.restore_database_dump(
+            database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock()
+        )
+
+
+def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
+    database_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        (
+            'pg_restore',
+            '--no-password',
+            '--if-exists',
+            '--exit-on-error',
+            '--clean',
+            '--dbname',
+            'foo',
             '--host',
             'database.example.org',
             '--port',
             '5433',
-            '--dbname',
-            'foo',
-            'databases/localhost/foo',
         ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
         extra_environment=None,
+        borg_local_path='borg',
     ).once()
     flexmock(module).should_receive('execute_command').with_args(
         (
@@ -255,30 +273,32 @@ def test_restore_database_dumps_runs_pg_restore_with_hostname_and_port():
         extra_environment=None,
     ).once()
 
-    module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)
-
-
-def test_restore_database_dumps_runs_pg_restore_with_username_and_password():
-    databases = [{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1'}]
-    flexmock(module).should_receive('make_dump_path').and_return('')
-    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
-        'databases/localhost/foo'
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
     )
 
-    flexmock(module).should_receive('execute_command').with_args(
+
+def test_restore_database_dump_runs_pg_restore_with_username_and_password():
+    database_config = [{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1'}]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
         (
             'pg_restore',
             '--no-password',
-            '--clean',
             '--if-exists',
             '--exit-on-error',
-            '--username',
-            'postgres',
+            '--clean',
             '--dbname',
             'foo',
-            'databases/localhost/foo',
+            '--username',
+            'postgres',
         ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
         extra_environment={'PGPASSWORD': 'trustsome1'},
+        borg_local_path='borg',
     ).once()
     flexmock(module).should_receive('execute_command').with_args(
         (
@@ -295,4 +315,37 @@ def test_restore_database_dumps_runs_pg_restore_with_username_and_password():
         extra_environment={'PGPASSWORD': 'trustsome1'},
     ).once()
 
-    module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+    )
+
+
+def test_restore_database_dump_runs_psql_for_all_database_dump():
+    database_config = [{'name': 'all'}]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        ('psql', '--no-password'),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment=None,
+        borg_local_path='borg',
+    ).once()
+    flexmock(module).should_receive('execute_command').with_args(
+        ('psql', '--no-password', '--quiet', '--command', 'ANALYZE'), extra_environment=None
+    ).once()
+
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+    )
+
+
+def test_restore_database_dump_with_dry_run_skips_restore():
+    database_config = [{'name': 'foo'}]
+
+    flexmock(module).should_receive('execute_command_with_processes').never()
+
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock()
+    )

tests/unit/test_execute.py

@@ -1,3 +1,5 @@
+import subprocess
+
 import pytest
 from flexmock import flexmock
 
@@ -5,24 +7,56 @@ from borgmatic import execute as module
 
 
 @pytest.mark.parametrize(
-    'exit_code,error_on_warnings,expected_result',
+    'process,exit_code,borg_local_path,expected_result',
     (
-        (2, True, True),
-        (2, False, True),
-        (1, True, True),
-        (1, False, False),
-        (0, True, False),
-        (0, False, False),
+        (flexmock(args=['grep']), 2, None, True),
+        (flexmock(args=['grep']), 2, 'borg', True),
+        (flexmock(args=['borg']), 2, 'borg', True),
+        (flexmock(args=['borg1']), 2, 'borg1', True),
+        (flexmock(args=['grep']), 1, None, True),
+        (flexmock(args=['grep']), 1, 'borg', True),
+        (flexmock(args=['borg']), 1, 'borg', False),
+        (flexmock(args=['borg1']), 1, 'borg1', False),
+        (flexmock(args=['grep']), 0, None, False),
+        (flexmock(args=['grep']), 0, 'borg', False),
+        (flexmock(args=['borg']), 0, 'borg', False),
+        (flexmock(args=['borg1']), 0, 'borg1', False),
+        (flexmock(args=['borg']), None, None, False),
     ),
 )
-def test_exit_code_indicates_error_respects_exit_code_and_error_on_warnings(
-    exit_code, error_on_warnings, expected_result
+def test_exit_code_indicates_error_respects_exit_code_and_borg_local_path(
+    process, exit_code, borg_local_path, expected_result
 ):
+    assert module.exit_code_indicates_error(process, exit_code, borg_local_path) is expected_result
+
+
+def test_command_for_process_converts_sequence_command_to_string():
+    process = flexmock(args=['foo', 'bar', 'baz'])
+
+    assert module.command_for_process(process) == 'foo bar baz'
+
+
+def test_command_for_process_passes_through_string_command():
+    process = flexmock(args='foo bar baz')
+
+    assert module.command_for_process(process) == 'foo bar baz'
+
+
+def test_output_buffer_for_process_returns_stderr_when_stdout_excluded():
+    stdout = flexmock()
+    stderr = flexmock()
+    process = flexmock(stdout=stdout, stderr=stderr)
+
+    assert module.output_buffer_for_process(process, exclude_stdouts=[flexmock(), stdout]) == stderr
+
+
+def test_output_buffer_for_process_returns_stdout_when_not_excluded():
+    stdout = flexmock()
+    process = flexmock(stdout=stdout)
+
     assert (
-        module.exit_code_indicates_error(
-            ('command',), exit_code, error_on_warnings=error_on_warnings
-        )
-        is expected_result
+        module.output_buffer_for_process(process, exclude_stdouts=[flexmock(), flexmock()])
+        == stdout
     )
 
 
@@ -38,7 +72,7 @@ def test_execute_command_calls_full_command():
         env=None,
         cwd=None,
     ).and_return(flexmock(stdout=None)).once()
-    flexmock(module).should_receive('log_output')
+    flexmock(module).should_receive('log_outputs')
 
     output = module.execute_command(full_command)
 
@@ -58,13 +92,27 @@ def test_execute_command_calls_full_command_with_output_file():
         env=None,
         cwd=None,
     ).and_return(flexmock(stderr=None)).once()
-    flexmock(module).should_receive('log_output')
+    flexmock(module).should_receive('log_outputs')
 
     output = module.execute_command(full_command, output_file=output_file)
 
     assert output is None
 
 
+def test_execute_command_calls_full_command_without_capturing_output():
+    full_command = ['foo', 'bar']
+    flexmock(module.os, environ={'a': 'b'})
+    flexmock(module.subprocess).should_receive('Popen').with_args(
+        full_command, stdin=None, stdout=None, stderr=None, shell=False, env=None, cwd=None
+    ).and_return(flexmock(wait=lambda: 0)).once()
+    flexmock(module).should_receive('exit_code_indicates_error').and_return(False)
+    flexmock(module).should_receive('log_outputs')
+
+    output = module.execute_command(full_command, output_file=module.DO_NOT_CAPTURE)
+
+    assert output is None
+
+
 def test_execute_command_calls_full_command_with_input_file():
     full_command = ['foo', 'bar']
     input_file = flexmock(name='test')
@@ -78,7 +126,7 @@ def test_execute_command_calls_full_command_with_input_file():
         env=None,
         cwd=None,
     ).and_return(flexmock(stdout=None)).once()
-    flexmock(module).should_receive('log_output')
+    flexmock(module).should_receive('log_outputs')
 
     output = module.execute_command(full_command, input_file=input_file)
 
@@ -89,7 +137,7 @@ def test_execute_command_calls_full_command_with_shell():
     full_command = ['foo', 'bar']
     flexmock(module.os, environ={'a': 'b'})
     flexmock(module.subprocess).should_receive('Popen').with_args(
-        full_command,
+        ' '.join(full_command),
         stdin=None,
         stdout=module.subprocess.PIPE,
         stderr=module.subprocess.STDOUT,
@@ -97,7 +145,7 @@ def test_execute_command_calls_full_command_with_shell():
         env=None,
         cwd=None,
     ).and_return(flexmock(stdout=None)).once()
-    flexmock(module).should_receive('log_output')
+    flexmock(module).should_receive('log_outputs')
 
     output = module.execute_command(full_command, shell=True)
 
@@ -116,7 +164,7 @@ def test_execute_command_calls_full_command_with_extra_environment():
         env={'a': 'b', 'c': 'd'},
         cwd=None,
     ).and_return(flexmock(stdout=None)).once()
-    flexmock(module).should_receive('log_output')
+    flexmock(module).should_receive('log_outputs')
 
     output = module.execute_command(full_command, extra_environment={'c': 'd'})
 
@@ -135,13 +183,31 @@ def test_execute_command_calls_full_command_with_working_directory():
         env=None,
         cwd='/working',
     ).and_return(flexmock(stdout=None)).once()
-    flexmock(module).should_receive('log_output')
+    flexmock(module).should_receive('log_outputs')
 
     output = module.execute_command(full_command, working_directory='/working')
 
     assert output is None
 
 
+def test_execute_command_without_run_to_completion_returns_process():
+    full_command = ['foo', 'bar']
+    process = flexmock()
+    flexmock(module.os, environ={'a': 'b'})
+    flexmock(module.subprocess).should_receive('Popen').with_args(
+        full_command,
+        stdin=None,
+        stdout=module.subprocess.PIPE,
+        stderr=module.subprocess.STDOUT,
+        shell=False,
+        env=None,
+        cwd=None,
+    ).and_return(process).once()
+    flexmock(module).should_receive('log_outputs')
+
+    assert module.execute_command(full_command, run_to_completion=False) == process
+
+
 def test_execute_command_captures_output():
     full_command = ['foo', 'bar']
     expected_output = '[]'
@@ -160,7 +226,7 @@ def test_execute_command_captures_output_with_shell():
     expected_output = '[]'
     flexmock(module.os, environ={'a': 'b'})
     flexmock(module.subprocess).should_receive('check_output').with_args(
-        full_command, shell=True, env=None, cwd=None
+        'foo bar', shell=True, env=None, cwd=None
     ).and_return(flexmock(decode=lambda: expected_output)).once()
 
     output = module.execute_command(full_command, output_log_level=None, shell=True)
@@ -198,28 +264,166 @@ def test_execute_command_captures_output_with_working_directory():
     assert output == expected_output
 
 
-def test_execute_command_without_capture_does_not_raise_on_success():
-    flexmock(module.subprocess).should_receive('check_call').and_raise(
-        module.subprocess.CalledProcessError(0, 'borg init')
-    )
+def test_execute_command_with_processes_calls_full_command():
+    full_command = ['foo', 'bar']
+    processes = (flexmock(),)
+    flexmock(module.os, environ={'a': 'b'})
+    flexmock(module.subprocess).should_receive('Popen').with_args(
+        full_command,
+        stdin=None,
+        stdout=module.subprocess.PIPE,
+        stderr=module.subprocess.STDOUT,
+        shell=False,
+        env=None,
+        cwd=None,
+    ).and_return(flexmock(stdout=None)).once()
+    flexmock(module).should_receive('log_outputs')
 
-    module.execute_command_without_capture(('borg', 'init'))
+    output = module.execute_command_with_processes(full_command, processes)
+
+    assert output is None
 
 
-def test_execute_command_without_capture_does_not_raise_on_warning():
+def test_execute_command_with_processes_calls_full_command_with_output_file():
+    full_command = ['foo', 'bar']
+    processes = (flexmock(),)
+    output_file = flexmock(name='test')
+    flexmock(module.os, environ={'a': 'b'})
+    flexmock(module.subprocess).should_receive('Popen').with_args(
+        full_command,
+        stdin=None,
+        stdout=output_file,
+        stderr=module.subprocess.PIPE,
+        shell=False,
+        env=None,
+        cwd=None,
+    ).and_return(flexmock(stderr=None)).once()
+    flexmock(module).should_receive('log_outputs')
+
+    output = module.execute_command_with_processes(full_command, processes, output_file=output_file)
+
+    assert output is None
+
+
+def test_execute_command_with_processes_calls_full_command_without_capturing_output():
+    full_command = ['foo', 'bar']
+    processes = (flexmock(),)
+    flexmock(module.os, environ={'a': 'b'})
+    flexmock(module.subprocess).should_receive('Popen').with_args(
+        full_command, stdin=None, stdout=None, stderr=None, shell=False, env=None, cwd=None
+    ).and_return(flexmock(wait=lambda: 0)).once()
     flexmock(module).should_receive('exit_code_indicates_error').and_return(False)
-    flexmock(module.subprocess).should_receive('check_call').and_raise(
-        module.subprocess.CalledProcessError(1, 'borg init')
+    flexmock(module).should_receive('log_outputs')
+
+    output = module.execute_command_with_processes(
+        full_command, processes, output_file=module.DO_NOT_CAPTURE
     )
 
-    module.execute_command_without_capture(('borg', 'init'))
+    assert output is None
 
 
-def test_execute_command_without_capture_raises_on_error():
-    flexmock(module).should_receive('exit_code_indicates_error').and_return(True)
-    flexmock(module.subprocess).should_receive('check_call').and_raise(
-        module.subprocess.CalledProcessError(2, 'borg init')
+def test_execute_command_with_processes_calls_full_command_with_input_file():
+    full_command = ['foo', 'bar']
+    processes = (flexmock(),)
+    input_file = flexmock(name='test')
+    flexmock(module.os, environ={'a': 'b'})
+    flexmock(module.subprocess).should_receive('Popen').with_args(
+        full_command,
+        stdin=input_file,
+        stdout=module.subprocess.PIPE,
+        stderr=module.subprocess.STDOUT,
+        shell=False,
+        env=None,
+        cwd=None,
+    ).and_return(flexmock(stdout=None)).once()
+    flexmock(module).should_receive('log_outputs')
+
+    output = module.execute_command_with_processes(full_command, processes, input_file=input_file)
+
+    assert output is None
+
+
+def test_execute_command_with_processes_calls_full_command_with_shell():
+    full_command = ['foo', 'bar']
+    processes = (flexmock(),)
+    flexmock(module.os, environ={'a': 'b'})
+    flexmock(module.subprocess).should_receive('Popen').with_args(
+        ' '.join(full_command),
+        stdin=None,
+        stdout=module.subprocess.PIPE,
+        stderr=module.subprocess.STDOUT,
+        shell=True,
+        env=None,
+        cwd=None,
+    ).and_return(flexmock(stdout=None)).once()
+    flexmock(module).should_receive('log_outputs')
+
+    output = module.execute_command_with_processes(full_command, processes, shell=True)
+
+    assert output is None
+
+
+def test_execute_command_with_processes_calls_full_command_with_extra_environment():
+    full_command = ['foo', 'bar']
+    processes = (flexmock(),)
+    flexmock(module.os, environ={'a': 'b'})
+    flexmock(module.subprocess).should_receive('Popen').with_args(
+        full_command,
+        stdin=None,
+        stdout=module.subprocess.PIPE,
+        stderr=module.subprocess.STDOUT,
+        shell=False,
+        env={'a': 'b', 'c': 'd'},
+        cwd=None,
+    ).and_return(flexmock(stdout=None)).once()
+    flexmock(module).should_receive('log_outputs')
+
+    output = module.execute_command_with_processes(
+        full_command, processes, extra_environment={'c': 'd'}
     )
 
-    with pytest.raises(module.subprocess.CalledProcessError):
-        module.execute_command_without_capture(('borg', 'init'))
+    assert output is None
+
+
+def test_execute_command_with_processes_calls_full_command_with_working_directory():
+    full_command = ['foo', 'bar']
+    processes = (flexmock(),)
+    flexmock(module.os, environ={'a': 'b'})
+    flexmock(module.subprocess).should_receive('Popen').with_args(
+        full_command,
+        stdin=None,
+        stdout=module.subprocess.PIPE,
+        stderr=module.subprocess.STDOUT,
+        shell=False,
+        env=None,
+        cwd='/working',
+    ).and_return(flexmock(stdout=None)).once()
+    flexmock(module).should_receive('log_outputs')
+
+    output = module.execute_command_with_processes(
+        full_command, processes, working_directory='/working'
+    )
+
+    assert output is None
+
+
+def test_execute_command_with_processes_kills_processes_on_error():
+    full_command = ['foo', 'bar']
+    process = flexmock(stdout=flexmock(read=lambda count: None))
+    process.should_receive('poll')
+    process.should_receive('kill').once()
+    processes = (process,)
+    flexmock(module.os, environ={'a': 'b'})
+    flexmock(module.subprocess).should_receive('Popen').with_args(
+        full_command,
+        stdin=None,
+        stdout=module.subprocess.PIPE,
+        stderr=module.subprocess.STDOUT,
+        shell=False,
+        env=None,
+        cwd=None,
+    ).and_raise(subprocess.CalledProcessError(1, full_command, 'error')).once()
+    flexmock(module).should_receive('log_outputs').never()
+
+    with pytest.raises(subprocess.CalledProcessError):
+        module.execute_command_with_processes(full_command, processes)