Bump version for release.

NEWS

@@ -1,3 +1,42 @@
+1.5.6
+ * #292: Allow before_backup and similiar hooks to exit with a soft failure without altering the
+   monitoring status on Healthchecks or other providers. Support this by waiting to ping monitoring
+   services with a "start" status until after before_* hooks finish. Failures in before_* hooks
+   still trigger a monitoring "fail" status.
+ * #316: Fix hang when a stale database dump named pipe from an aborted borgmatic run remains on
+   disk.
+ * #323: Fix for certain configuration options like ssh_command impacting Borg invocations for
+   separate configuration files.
+ * #324: Add "borgmatic extract --strip-components" flag to remove leading path components when
+   extracting an archive.
+ * Tweak comment indentation in generated configuration file for clarity.
+ * Link to Borgmacator GNOME AppIndicator from monitoring documentation.
+
+1.5.5
+ * #314: Fix regression in support for PostgreSQL's "directory" dump format. Unlike other dump
+   formats, the "directory" dump format does not stream directly to/from Borg.
+ * #315: Fix enabled database hooks to implicitly set one_file_system configuration option to true.
+   This prevents Borg from reading devices like /dev/zero and hanging.
+ * #316: Fix hang when streaming a database dump to Borg with implicit duplicate source directories
+   by deduplicating them first.
+ * #319: Fix error message when there are no MySQL databases to dump for "all" databases.
+ * Improve documentation around the installation process. Specifically, making borgmatic commands
+   runnable via the system PATH and offering a global install option.
+
+1.5.4
+ * #310: Fix legitimate database dump command errors (exit code 1) not being treated as errors by
+   borgmatic.
+ * For database dumps, replace the named pipe on every borgmatic run. This prevent hangs on stale
+   pipes left over from previous runs.
+ * Fix error handling to handle more edge cases when executing commands.
+
+1.5.3
+ * #258: Stream database dumps and restores directly to/from Borg without using any additional
+   filesystem space. This feature is automatic, and works even on restores from archives made with
+   previous versions of borgmatic.
+ * #293: Documentation on macOS launchd permissions issues with work-around for Full Disk Access.
+ * Remove "borgmatic restore --progress" flag, as it now conflicts with streaming database restores.
+
 1.5.2
  * #301: Fix MySQL restore error on "all" database dump by excluding system tables.
  * Fix PostgreSQL restore error on "all" database dump by using "psql" for the restore instead of

borgmatic/borg/check.py

@@ -1,7 +1,7 @@
 import logging
 
 from borgmatic.borg import extract
-from borgmatic.execute import execute_command, execute_command_without_capture
+from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 DEFAULT_CHECKS = ('repository', 'archives')
 DEFAULT_PREFIX = '{hostname}-'
@@ -134,9 +134,9 @@ def check_archives(
         # The Borg repair option trigger an interactive prompt, which won't work when output is
         # captured. And progress messes with the terminal directly.
         if repair or progress:
-            execute_command_without_capture(full_command, error_on_warnings=True)
+            execute_command(full_command, output_file=DO_NOT_CAPTURE)
         else:
-            execute_command(full_command, error_on_warnings=True)
+            execute_command(full_command)
 
     if 'extract' in checks:
         extract.extract_last_archive_dry_run(repository, lock_wait, local_path, remote_path)

borgmatic/borg/create.py

@@ -2,9 +2,10 @@ import glob
 import itertools
 import logging
 import os
+import pathlib
 import tempfile
 
-from borgmatic.execute import execute_command, execute_command_without_capture
+from borgmatic.execute import DO_NOT_CAPTURE, execute_command, execute_command_with_processes
 
 logger = logging.getLogger(__name__)
 
@@ -43,6 +44,35 @@ def _expand_home_directories(directories):
     return tuple(os.path.expanduser(directory) for directory in directories)
 
 
+def deduplicate_directories(directories):
+    '''
+    Given a sequence of directories, return them as a sorted tuple with all duplicate child
+    directories removed. For instance, if paths is ('/foo', '/foo/bar'), return just: ('/foo',)
+
+    The idea is that if Borg is given a parent directory, then it doesn't also need to be given
+    child directories, because it will naturally spider the contents of the parent directory. And
+    there are cases where Borg coming across the same file twice will result in duplicate reads and
+    even hangs, e.g. when a database hook is using a named pipe for streaming database dumps to
+    Borg.
+    '''
+    deduplicated = set()
+
+    for directory in sorted(directories):
+        # If the directory is "/", that contains all child directories, so we can early out.
+        if directory == os.path.sep:
+            return (os.path.sep,)
+
+        # If no other directories are parents of current directory (even n levels up), then the
+        # current directory isn't a duplicate.
+        if not any(
+            pathlib.PurePath(other_directory) in pathlib.PurePath(directory).parents
+            for other_directory in directories
+        ):
+            deduplicated.add(directory)
+
+    return tuple(sorted(deduplicated))
+
+
 def _write_pattern_file(patterns=None):
     '''
     Given a sequence of patterns, write them to a named temporary file and return it. Return None
@@ -125,6 +155,9 @@ def borgmatic_source_directories(borgmatic_source_directory):
     )
 
 
+DEFAULT_ARCHIVE_NAME_FORMAT = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
+
+
 def create_archive(
     dry_run,
     repository,
@@ -136,14 +169,20 @@ def create_archive(
     stats=False,
     json=False,
     files=False,
+    stream_processes=None,
 ):
     '''
     Given vebosity/dry-run flags, a local or remote repository path, a location config dict, and a
     storage config dict, create a Borg archive and return Borg's JSON output (if any).
+
+    If a sequence of stream processes is given (instances of subprocess.Popen), then execute the
+    create command while also triggering the given processes to produce output.
     '''
-    sources = _expand_directories(
-        location_config['source_directories']
-        + borgmatic_source_directories(location_config.get('borgmatic_source_directory'))
+    sources = deduplicate_directories(
+        _expand_directories(
+            location_config['source_directories']
+            + borgmatic_source_directories(location_config.get('borgmatic_source_directory'))
+        )
     )
 
     pattern_file = _write_pattern_file(location_config.get('patterns'))
@@ -157,8 +196,7 @@ def create_archive(
     umask = storage_config.get('umask', None)
     lock_wait = storage_config.get('lock_wait', None)
     files_cache = location_config.get('files_cache')
-    default_archive_name_format = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
-    archive_name_format = storage_config.get('archive_name_format', default_archive_name_format)
+    archive_name_format = storage_config.get('archive_name_format', DEFAULT_ARCHIVE_NAME_FORMAT)
     extra_borg_options = storage_config.get('extra_borg_options', {}).get('create', '')
 
     full_command = (
@@ -169,12 +207,16 @@ def create_archive(
         + (('--chunker-params', chunker_params) if chunker_params else ())
         + (('--compression', compression) if compression else ())
         + (('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ())
-        + (('--one-file-system',) if location_config.get('one_file_system') else ())
+        + (
+            ('--one-file-system',)
+            if location_config.get('one_file_system') or stream_processes
+            else ()
+        )
         + (('--numeric-owner',) if location_config.get('numeric_owner') else ())
         + (('--noatime',) if location_config.get('atime') is False else ())
         + (('--noctime',) if location_config.get('ctime') is False else ())
         + (('--nobirthtime',) if location_config.get('birthtime') is False else ())
-        + (('--read-special',) if location_config.get('read_special') else ())
+        + (('--read-special',) if (location_config.get('read_special') or stream_processes) else ())
         + (('--nobsdflags',) if location_config.get('bsd_flags') is False else ())
         + (('--files-cache', files_cache) if files_cache else ())
         + (('--remote-path', remote_path) if remote_path else ())
@@ -196,12 +238,6 @@ def create_archive(
         + sources
     )
 
-    # The progress output isn't compatible with captured and logged output, as progress messes with
-    # the terminal directly.
-    if progress:
-        execute_command_without_capture(full_command, error_on_warnings=False)
-        return
-
     if json:
         output_log_level = None
     elif (stats or files) and logger.getEffectiveLevel() == logging.WARNING:
@@ -209,4 +245,17 @@ def create_archive(
     else:
         output_log_level = logging.INFO
 
-    return execute_command(full_command, output_log_level, error_on_warnings=False)
+    # The progress output isn't compatible with captured and logged output, as progress messes with
+    # the terminal directly.
+    output_file = DO_NOT_CAPTURE if progress else None
+
+    if stream_processes:
+        return execute_command_with_processes(
+            full_command,
+            stream_processes,
+            output_log_level,
+            output_file,
+            borg_local_path=local_path,
+        )
+
+    return execute_command(full_command, output_log_level, output_file, borg_local_path=local_path)

borgmatic/borg/environment.py

@@ -22,6 +22,8 @@ def initialize(storage_config):
         value = storage_config.get(option_name)
         if value:
             os.environ[environment_variable_name] = value
+        else:
+            os.environ.pop(environment_variable_name, None)
 
     for (
         option_name,

borgmatic/borg/extract.py

@@ -1,7 +1,8 @@
 import logging
 import os
+import subprocess
 
-from borgmatic.execute import execute_command, execute_command_without_capture
+from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 logger = logging.getLogger(__name__)
 
@@ -27,7 +28,9 @@ def extract_last_archive_dry_run(repository, lock_wait=None, local_path='borg',
         + (repository,)
     )
 
-    list_output = execute_command(full_list_command, output_log_level=None, error_on_warnings=False)
+    list_output = execute_command(
+        full_list_command, output_log_level=None, borg_local_path=local_path
+    )
 
     try:
         last_archive_name = list_output.strip().splitlines()[-1]
@@ -48,7 +51,7 @@ def extract_last_archive_dry_run(repository, lock_wait=None, local_path='borg',
         )
     )
 
-    execute_command(full_extract_command, working_directory=None, error_on_warnings=True)
+    execute_command(full_extract_command, working_directory=None)
 
 
 def extract_archive(
@@ -61,18 +64,25 @@ def extract_archive(
     local_path='borg',
     remote_path=None,
     destination_path=None,
+    strip_components=None,
     progress=False,
-    error_on_warnings=True,
+    extract_to_stdout=False,
 ):
     '''
     Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to
     restore from the archive, location/storage configuration dicts, optional local and remote Borg
     paths, and an optional destination path to extract to, extract the archive into the current
     directory.
+
+    If extract to stdout is True, then start the extraction streaming to stdout, and return that
+    extract process as an instance of subprocess.Popen.
     '''
     umask = storage_config.get('umask', None)
     lock_wait = storage_config.get('lock_wait', None)
 
+    if progress and extract_to_stdout:
+        raise ValueError('progress and extract_to_stdout cannot both be set')
+
     full_command = (
         (local_path, 'extract')
         + (('--remote-path', remote_path) if remote_path else ())
@@ -82,7 +92,9 @@ def extract_archive(
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--dry-run',) if dry_run else ())
+        + (('--strip-components', str(strip_components)) if strip_components else ())
         + (('--progress',) if progress else ())
+        + (('--stdout',) if extract_to_stdout else ())
         + ('::'.join((repository if ':' in repository else os.path.abspath(repository), archive)),)
         + (tuple(paths) if paths else ())
     )
@@ -90,13 +102,19 @@ def extract_archive(
     # The progress output isn't compatible with captured and logged output, as progress messes with
     # the terminal directly.
     if progress:
-        execute_command_without_capture(
-            full_command, working_directory=destination_path, error_on_warnings=error_on_warnings
+        return execute_command(
+            full_command, output_file=DO_NOT_CAPTURE, working_directory=destination_path
         )
-        return
+        return None
 
-    # Error on warnings by default, as Borg only gives a warning if the restore paths don't exist in
-    # the archive!
-    execute_command(
-        full_command, working_directory=destination_path, error_on_warnings=error_on_warnings
-    )
+    if extract_to_stdout:
+        return execute_command(
+            full_command,
+            output_file=subprocess.PIPE,
+            working_directory=destination_path,
+            run_to_completion=False,
+        )
+
+    # Don't give Borg local path, so as to error on warnings, as Borg only gives a warning if the
+    # restore paths don't exist in the archive!
+    execute_command(full_command, working_directory=destination_path)

borgmatic/borg/info.py

@@ -41,5 +41,5 @@ def display_archives_info(
     return execute_command(
         full_command,
         output_log_level=None if info_arguments.json else logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path=local_path,
     )

borgmatic/borg/init.py

@@ -1,7 +1,7 @@
 import logging
 import subprocess
 
-from borgmatic.execute import execute_command, execute_command_without_capture
+from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 logger = logging.getLogger(__name__)
 
@@ -54,5 +54,5 @@ def initialize_repository(
         + (repository,)
     )
 
-    # Don't use execute_command() here because it doesn't support interactive prompts.
-    execute_command_without_capture(init_command, error_on_warnings=False)
+    # Do not capture output here, so as to support interactive prompts.
+    execute_command(init_command, output_file=DO_NOT_CAPTURE, borg_local_path=local_path)

borgmatic/borg/list.py

@@ -36,7 +36,7 @@ def resolve_archive_name(repository, archive, storage_config, local_path='borg',
         + ('--short', repository)
     )
 
-    output = execute_command(full_command, output_log_level=None, error_on_warnings=False)
+    output = execute_command(full_command, output_log_level=None, borg_local_path=local_path)
     try:
         latest_archive = output.strip().splitlines()[-1]
     except IndexError:
@@ -85,5 +85,5 @@ def list_archives(repository, storage_config, list_arguments, local_path='borg',
     return execute_command(
         full_command,
         output_log_level=None if list_arguments.json else logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path=local_path,
     )

borgmatic/borg/mount.py

@@ -1,6 +1,6 @@
 import logging
 
-from borgmatic.execute import execute_command, execute_command_without_capture
+from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 logger = logging.getLogger(__name__)
 
@@ -40,7 +40,7 @@ def mount_archive(
 
     # Don't capture the output when foreground mode is used so that ctrl-C can work properly.
     if foreground:
-        execute_command_without_capture(full_command, error_on_warnings=False)
+        execute_command(full_command, output_file=DO_NOT_CAPTURE, borg_local_path=local_path)
         return
 
-    execute_command(full_command, error_on_warnings=False)
+    execute_command(full_command, borg_local_path=local_path)

borgmatic/borg/prune.py

@@ -72,4 +72,4 @@ def prune_archives(
     else:
         output_log_level = logging.INFO
 
-    execute_command(full_command, output_log_level=output_log_level, error_on_warnings=False)
+    execute_command(full_command, output_log_level=output_log_level, borg_local_path=local_path)

borgmatic/borg/umount.py

@@ -17,4 +17,4 @@ def unmount_archive(mount_point, local_path='borg'):
         + (mount_point,)
     )
 
-    execute_command(full_command, error_on_warnings=True)
+    execute_command(full_command)

borgmatic/commands/arguments.py

@@ -340,6 +340,13 @@ def parse_arguments(*unparsed_arguments):
         dest='destination',
         help='Directory to extract files into, defaults to the current directory',
     )
+    extract_group.add_argument(
+        '--strip-components',
+        type=int,
+        metavar='NUMBER',
+        dest='strip_components',
+        help='Number of leading path components to remove from each extracted path. Skip paths with fewer elements',
+    )
     extract_group.add_argument(
         '--progress',
         dest='progress',
@@ -427,13 +434,6 @@ def parse_arguments(*unparsed_arguments):
         dest='databases',
         help='Names of databases to restore from archive, defaults to all databases. Note that any databases to restore must be defined in borgmatic\'s configuration',
     )
-    restore_group.add_argument(
-        '--progress',
-        dest='progress',
-        default=False,
-        action='store_true',
-        help='Display progress for each database dump file as it is extracted from archive',
-    )
     restore_group.add_argument(
         '-h', '--help', action='help', help='Show this help message and exit'
     )

borgmatic/commands/borgmatic.py

@@ -59,11 +59,10 @@ def run_configuration(config_filename, config, arguments):
     try:
         if prune_create_or_check:
             dispatch.call_hooks(
-                'ping_monitor',
+                'initialize_monitor',
                 hooks,
                 config_filename,
                 monitor.MONITOR_HOOK_NAMES,
-                monitor.State.START,
                 monitoring_log_level,
                 global_arguments.dry_run,
             )
@@ -83,14 +82,6 @@ def run_configuration(config_filename, config, arguments):
                 'pre-backup',
                 global_arguments.dry_run,
             )
-            dispatch.call_hooks(
-                'dump_databases',
-                hooks,
-                config_filename,
-                dump.DATABASE_HOOK_NAMES,
-                location,
-                global_arguments.dry_run,
-            )
         if 'check' in arguments:
             command.execute_hook(
                 hooks.get('before_check'),
@@ -99,6 +90,16 @@ def run_configuration(config_filename, config, arguments):
                 'pre-check',
                 global_arguments.dry_run,
             )
+        if prune_create_or_check:
+            dispatch.call_hooks(
+                'ping_monitor',
+                hooks,
+                config_filename,
+                monitor.MONITOR_HOOK_NAMES,
+                monitor.State.START,
+                monitoring_log_level,
+                global_arguments.dry_run,
+            )
     except (OSError, CalledProcessError) as error:
         if command.considered_soft_failure(config_filename, error):
             return
@@ -131,6 +132,16 @@ def run_configuration(config_filename, config, arguments):
 
     if not encountered_error:
         try:
+            if prune_create_or_check:
+                dispatch.call_hooks(
+                    'ping_monitor',
+                    hooks,
+                    config_filename,
+                    monitor.MONITOR_HOOK_NAMES,
+                    monitor.State.FINISH,
+                    monitoring_log_level,
+                    global_arguments.dry_run,
+                )
             if 'prune' in arguments:
                 command.execute_hook(
                     hooks.get('after_prune'),
@@ -163,16 +174,6 @@ def run_configuration(config_filename, config, arguments):
                     'post-check',
                     global_arguments.dry_run,
                 )
-            if {'prune', 'create', 'check'}.intersection(arguments):
-                dispatch.call_hooks(
-                    'ping_monitor',
-                    hooks,
-                    config_filename,
-                    monitor.MONITOR_HOOK_NAMES,
-                    monitor.State.FINISH,
-                    monitoring_log_level,
-                    global_arguments.dry_run,
-                )
         except (OSError, CalledProcessError) as error:
             if command.considered_soft_failure(config_filename, error):
                 return
@@ -184,6 +185,15 @@ def run_configuration(config_filename, config, arguments):
 
     if encountered_error and prune_create_or_check:
         try:
+            dispatch.call_hooks(
+                'ping_monitor',
+                hooks,
+                config_filename,
+                monitor.MONITOR_HOOK_NAMES,
+                monitor.State.FAIL,
+                monitoring_log_level,
+                global_arguments.dry_run,
+            )
             command.execute_hook(
                 hooks.get('on_error'),
                 hooks.get('umask'),
@@ -194,15 +204,6 @@ def run_configuration(config_filename, config, arguments):
                 error=encountered_error,
                 output=getattr(encountered_error, 'output', ''),
             )
-            dispatch.call_hooks(
-                'ping_monitor',
-                hooks,
-                config_filename,
-                monitor.MONITOR_HOOK_NAMES,
-                monitor.State.FAIL,
-                monitoring_log_level,
-                global_arguments.dry_run,
-            )
         except (OSError, CalledProcessError) as error:
             if command.considered_soft_failure(config_filename, error):
                 return
@@ -262,6 +263,24 @@ def run_actions(
         )
     if 'create' in arguments:
         logger.info('{}: Creating archive{}'.format(repository, dry_run_label))
+        dispatch.call_hooks(
+            'remove_database_dumps',
+            hooks,
+            repository,
+            dump.DATABASE_HOOK_NAMES,
+            location,
+            global_arguments.dry_run,
+        )
+        active_dumps = dispatch.call_hooks(
+            'dump_databases',
+            hooks,
+            repository,
+            dump.DATABASE_HOOK_NAMES,
+            location,
+            global_arguments.dry_run,
+        )
+        stream_processes = [process for processes in active_dumps.values() for process in processes]
+
         json_output = borg_create.create_archive(
             global_arguments.dry_run,
             repository,
@@ -273,9 +292,11 @@ def run_actions(
             stats=arguments['create'].stats,
             json=arguments['create'].json,
             files=arguments['create'].files,
+            stream_processes=stream_processes,
         )
         if json_output:
             yield json.loads(json_output)
+
     if 'check' in arguments and checks.repository_enabled_for_checks(repository, consistency):
         logger.info('{}: Running consistency checks'.format(repository))
         borg_check.check_archives(
@@ -307,6 +328,7 @@ def run_actions(
                 local_path=local_path,
                 remote_path=remote_path,
                 destination_path=arguments['extract'].destination,
+                strip_components=arguments['extract'].strip_components,
                 progress=arguments['extract'].progress,
             )
     if 'mount' in arguments:
@@ -342,62 +364,90 @@ def run_actions(
                     repository, arguments['restore'].archive
                 )
             )
+            dispatch.call_hooks(
+                'remove_database_dumps',
+                hooks,
+                repository,
+                dump.DATABASE_HOOK_NAMES,
+                location,
+                global_arguments.dry_run,
+            )
 
             restore_names = arguments['restore'].databases or []
             if 'all' in restore_names:
                 restore_names = []
 
-            # Extract dumps for the named databases from the archive.
-            dump_patterns = dispatch.call_hooks(
-                'make_database_dump_patterns',
+            archive_name = borg_list.resolve_archive_name(
+                repository, arguments['restore'].archive, storage, local_path, remote_path
+            )
+            found_names = set()
+
+            for hook_name, per_hook_restore_databases in hooks.items():
+                if hook_name not in dump.DATABASE_HOOK_NAMES:
+                    continue
+
+                for restore_database in per_hook_restore_databases:
+                    database_name = restore_database['name']
+                    if restore_names and database_name not in restore_names:
+                        continue
+
+                    found_names.add(database_name)
+                    dump_pattern = dispatch.call_hooks(
+                        'make_database_dump_pattern',
+                        hooks,
+                        repository,
+                        dump.DATABASE_HOOK_NAMES,
+                        location,
+                        database_name,
+                    )[hook_name]
+
+                    # Kick off a single database extract to stdout.
+                    extract_process = borg_extract.extract_archive(
+                        dry_run=global_arguments.dry_run,
+                        repository=repository,
+                        archive=archive_name,
+                        paths=dump.convert_glob_patterns_to_borg_patterns([dump_pattern]),
+                        location_config=location,
+                        storage_config=storage,
+                        local_path=local_path,
+                        remote_path=remote_path,
+                        destination_path='/',
+                        # A directory format dump isn't a single file, and therefore can't extract
+                        # to stdout. In this case, the extract_process return value is None.
+                        extract_to_stdout=bool(restore_database.get('format') != 'directory'),
+                    )
+
+                    # Run a single database restore, consuming the extract stdout (if any).
+                    dispatch.call_hooks(
+                        'restore_database_dump',
+                        {hook_name: [restore_database]},
+                        repository,
+                        dump.DATABASE_HOOK_NAMES,
+                        location,
+                        global_arguments.dry_run,
+                        extract_process,
+                    )
+
+            dispatch.call_hooks(
+                'remove_database_dumps',
                 hooks,
                 repository,
                 dump.DATABASE_HOOK_NAMES,
                 location,
-                restore_names,
-            )
-
-            borg_extract.extract_archive(
-                global_arguments.dry_run,
-                repository,
-                borg_list.resolve_archive_name(
-                    repository, arguments['restore'].archive, storage, local_path, remote_path
-                ),
-                dump.convert_glob_patterns_to_borg_patterns(
-                    dump.flatten_dump_patterns(dump_patterns, restore_names)
-                ),
-                location,
-                storage,
-                local_path=local_path,
-                remote_path=remote_path,
-                destination_path='/',
-                progress=arguments['restore'].progress,
-                # We don't want glob patterns that don't match to error.
-                error_on_warnings=False,
-            )
-
-            # Map the restore names or detected dumps to the corresponding database configurations.
-            restore_databases = dump.get_per_hook_database_configurations(
-                hooks, restore_names, dump_patterns
-            )
-
-            # Finally, restore the databases and cleanup the dumps.
-            dispatch.call_hooks(
-                'restore_database_dumps',
-                restore_databases,
-                repository,
-                dump.DATABASE_HOOK_NAMES,
-                location,
-                global_arguments.dry_run,
-            )
-            dispatch.call_hooks(
-                'remove_database_dumps',
-                restore_databases,
-                repository,
-                dump.DATABASE_HOOK_NAMES,
-                location,
                 global_arguments.dry_run,
             )
+
+            if not restore_names and not found_names:
+                raise ValueError('No databases were found to restore')
+
+            missing_names = sorted(set(restore_names) - found_names)
+            if missing_names:
+                raise ValueError(
+                    'Cannot restore database(s) {} missing from borgmatic\'s configuration'.format(
+                        ', '.join(missing_names)
+                    )
+                )
+
     if 'list' in arguments:
         if arguments['list'].repository is None or validate.repositories_match(
             repository, arguments['list'].repository

borgmatic/config/generate.py

@@ -37,9 +37,7 @@ def _schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
                 for item_schema in schema['seq']
             ]
         )
-        add_comments_to_configuration_sequence(
-            config, schema, indent=(level * INDENT) + SEQUENCE_INDENT
-        )
+        add_comments_to_configuration_sequence(config, schema, indent=(level * INDENT))
     elif 'map' in schema:
         config = yaml.comments.CommentedMap(
             [
@@ -86,8 +84,8 @@ def _comment_out_optional_configuration(rendered_config):
     optional = False
 
     for line in rendered_config.split('\n'):
-        # Upon encountering an optional configuration option, commenting out lines until the next
-        # blank line.
+        # Upon encountering an optional configuration option, comment out lines until the next blank
+        # line.
         if line.strip().startswith('# {}'.format(COMMENTED_OUT_SENTINEL)):
             optional = True
             continue
@@ -142,7 +140,7 @@ def add_comments_to_configuration_sequence(config, schema, indent=0):
 
     ```
     things:
-          # First key description. Added by this function.
+        # First key description. Added by this function.
         - key: foo
           # Second key description. Added by add_comments_to_configuration_map().
           other: bar

borgmatic/config/schema.yaml

@@ -30,7 +30,10 @@ map:
                     - user@backupserver:sourcehostname.borg
             one_file_system:
                 type: bool
-                desc: Stay in same file system (do not cross mount points). Defaults to false.
+                desc: |
+                  Stay in same file system (do not cross mount points). Defaults to false. But when
+                  a database hook is used, the setting here is ignored and one_file_system is
+                  considered true.
                 example: true
             numeric_owner:
                 type: bool
@@ -53,7 +56,9 @@ map:
                 desc: |
                     Use Borg's --read-special flag to allow backup of block and other special
                     devices. Use with caution, as it will lead to problems if used when
-                    backing up special devices such as /dev/zero. Defaults to false.
+                    backing up special devices such as /dev/zero. Defaults to false. But when a
+                    database hook is used, the setting here is ignored and read_special is
+                    considered true.
                 example: false
             bsd_flags:
                 type: bool
@@ -452,7 +457,9 @@ map:
                             type: str
                             desc: |
                                 Database name (required if using this hook). Or "all" to dump all
-                                databases on the host.
+                                databases on the host. Note that using this database hook implicitly
+                                enables both read_special and one_file_system (see above) to support
+                                dump and restore streaming.
                             example: users
                         hostname:
                             type: str
@@ -508,7 +515,9 @@ map:
                             type: str
                             desc: |
                                 Database name (required if using this hook). Or "all" to dump all
-                                databases on the host.
+                                databases on the host. Note that using this database hook implicitly
+                                enables both read_special and one_file_system (see above) to support
+                                dump and restore streaming.
                             example: users
                         hostname:
                             type: str

borgmatic/execute.py

@@ -1,5 +1,7 @@
+import collections
 import logging
 import os
+import select
 import subprocess
 
 logger = logging.getLogger(__name__)
@@ -9,52 +11,135 @@ ERROR_OUTPUT_MAX_LINE_COUNT = 25
 BORG_ERROR_EXIT_CODE = 2
 
 
-def exit_code_indicates_error(command, exit_code, error_on_warnings=True):
+def exit_code_indicates_error(process, exit_code, borg_local_path=None):
     '''
-    Return True if the given exit code from running the command corresponds to an error.
-    If error on warnings is False, then treat exit code 1 as a warning instead of an error.
+    Return True if the given exit code from running a command corresponds to an error. If a Borg
+    local path is given and matches the process' command, then treat exit code 1 as a warning
+    instead of an error.
     '''
-    if error_on_warnings:
-        return bool(exit_code != 0)
+    if exit_code is None:
+        return False
 
-    return bool(exit_code >= BORG_ERROR_EXIT_CODE)
+    command = process.args.split(' ') if isinstance(process.args, str) else process.args
+
+    if borg_local_path and command[0] == borg_local_path:
+        return bool(exit_code >= BORG_ERROR_EXIT_CODE)
+
+    return bool(exit_code != 0)
 
 
-def log_output(command, process, output_buffer, output_log_level, error_on_warnings):
+def command_for_process(process):
     '''
-    Given a command already executed, its process opened by subprocess.Popen(), and the process'
-    relevant output buffer (stderr or stdout), log its output with the requested log level.
-    Additionally, raise a CalledProcessException if the process exits with an error (or a warning,
-    if error on warnings is True).
+    Given a process as an instance of subprocess.Popen, return the command string that was used to
+    invoke it.
     '''
-    last_lines = []
+    return process.args if isinstance(process.args, str) else ' '.join(process.args)
 
-    while process.poll() is None:
-        line = output_buffer.readline().rstrip().decode()
-        if not line:
+
+def output_buffer_for_process(process, exclude_stdouts):
+    '''
+    Given a process as an instance of subprocess.Popen and a sequence of stdouts to exclude, return
+    either the process's stdout or stderr. The idea is that if stdout is excluded for a process, we
+    still have stderr to log.
+    '''
+    return process.stderr if process.stdout in exclude_stdouts else process.stdout
+
+
+def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
+    '''
+    Given a sequence of subprocess.Popen() instances for multiple processes, log the output for each
+    process with the requested log level. Additionally, raise a CalledProcessError if a process
+    exits with an error (or a warning for exit code 1, if that process matches the Borg local path).
+
+    For simplicity, it's assumed that the output buffer for each process is its stdout. But if any
+    stdouts are given to exclude, then for any matching processes, log from their stderr instead.
+
+    Note that stdout for a process can be None if output is intentionally not captured. In which
+    case it won't be logged.
+    '''
+    # Map from output buffer to sequence of last lines.
+    buffer_last_lines = collections.defaultdict(list)
+    output_buffers = [
+        output_buffer_for_process(process, exclude_stdouts)
+        for process in processes
+        if process.stdout or process.stderr
+    ]
+
+    # Log output for each process until they all exit.
+    while True:
+        if output_buffers:
+            (ready_buffers, _, _) = select.select(output_buffers, [], [])
+
+            for ready_buffer in ready_buffers:
+                line = ready_buffer.readline().rstrip().decode()
+                if not line:
+                    continue
+
+                # Keep the last few lines of output in case the process errors, and we need the output for
+                # the exception below.
+                last_lines = buffer_last_lines[ready_buffer]
+                last_lines.append(line)
+                if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT:
+                    last_lines.pop(0)
+
+                logger.log(output_log_level, line)
+
+        for process in processes:
+            exit_code = process.poll() if output_buffers else process.wait()
+
+            # If any process errors, then raise accordingly.
+            if exit_code_indicates_error(process, exit_code, borg_local_path):
+                # If an error occurs, include its output in the raised exception so that we don't
+                # inadvertently hide error output.
+                output_buffer = output_buffer_for_process(process, exclude_stdouts)
+
+                last_lines = buffer_last_lines[output_buffer] if output_buffer else []
+                if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT:
+                    last_lines.insert(0, '...')
+
+                # Something has gone wrong. So vent each process' output buffer to prevent it from
+                # hanging. And then kill the process.
+                for other_process in processes:
+                    if other_process.poll() is None:
+                        other_process.stdout.read(0)
+                        other_process.kill()
+
+                raise subprocess.CalledProcessError(
+                    exit_code, command_for_process(process), '\n'.join(last_lines)
+                )
+
+        if all(process.poll() is not None for process in processes):
+            break
+
+    # Consume any remaining output that we missed (if any).
+    for process in processes:
+        output_buffer = output_buffer_for_process(process, exclude_stdouts)
+
+        if not output_buffer:
             continue
 
-        # Keep the last few lines of output in case the command errors, and we need the output for
-        # the exception below.
-        last_lines.append(line)
-        if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT:
-            last_lines.pop(0)
+        remaining_output = output_buffer.read().rstrip().decode()
 
-        logger.log(output_log_level, line)
+        if remaining_output:  # pragma: no cover
+            logger.log(output_log_level, remaining_output)
 
-    remaining_output = output_buffer.read().rstrip().decode()
-    if remaining_output:  # pragma: no cover
-        logger.log(output_log_level, remaining_output)
 
-    exit_code = process.poll()
+def log_command(full_command, input_file, output_file):
+    '''
+    Log the given command (a sequence of command/argument strings), along with its input/output file
+    paths.
+    '''
+    logger.debug(
+        ' '.join(full_command)
+        + (' < {}'.format(getattr(input_file, 'name', '')) if input_file else '')
+        + (' > {}'.format(getattr(output_file, 'name', '')) if output_file else '')
+    )
 
-    if exit_code_indicates_error(command, exit_code, error_on_warnings):
-        # If an error occurs, include its output in the raised exception so that we don't
-        # inadvertently hide error output.
-        if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT:
-            last_lines.insert(0, '...')
 
-        raise subprocess.CalledProcessError(exit_code, ' '.join(command), '\n'.join(last_lines))
+# An sentinel passed as an output file to execute_command() to indicate that the command's output
+# should be allowed to flow through to stdout without being captured for logging. Useful for
+# commands with interactive prompts or those that mess directly with the console.
+DO_NOT_CAPTURE = object()
 
 
 def execute_command(
@@ -65,64 +150,108 @@ def execute_command(
     shell=False,
     extra_environment=None,
     working_directory=None,
-    error_on_warnings=True,
+    borg_local_path=None,
+    run_to_completion=True,
 ):
     '''
     Execute the given command (a sequence of command/argument strings) and log its output at the
-    given log level. If output log level is None, instead capture and return the output. If an
-    open output file object is given, then write stdout to the file and only log stderr (but only
-    if an output log level is set). If an open input file object is given, then read stdin from the
-    file. If shell is True, execute the command within a shell. If an extra environment dict is
-    given, then use it to augment the current environment, and pass the result into the command. If
-    a working directory is given, use that as the present working directory when running the
-    command. If error on warnings is False, then treat exit code 1 as a warning instead of an error.
+    given log level. If output log level is None, instead capture and return the output. (Implies
+    run_to_completion.) If an open output file object is given, then write stdout to the file and
+    only log stderr (but only if an output log level is set). If an open input file object is given,
+    then read stdin from the file. If shell is True, execute the command within a shell. If an extra
+    environment dict is given, then use it to augment the current environment, and pass the result
+    into the command. If a working directory is given, use that as the present working directory
+    when running the command. If a Borg local path is given, and the command matches it (regardless
+    of arguments), treat exit code 1 as a warning instead of an error. If run to completion is
+    False, then return the process for the command without executing it to completion.
 
     Raise subprocesses.CalledProcessError if an error occurs while running the command.
     '''
-    logger.debug(
-        ' '.join(full_command)
-        + (' < {}'.format(input_file.name) if input_file else '')
-        + (' > {}'.format(output_file.name) if output_file else '')
-    )
+    log_command(full_command, input_file, output_file)
     environment = {**os.environ, **extra_environment} if extra_environment else None
+    do_not_capture = bool(output_file is DO_NOT_CAPTURE)
+    command = ' '.join(full_command) if shell else full_command
 
     if output_log_level is None:
         output = subprocess.check_output(
-            full_command, shell=shell, env=environment, cwd=working_directory
+            command, shell=shell, env=environment, cwd=working_directory
         )
         return output.decode() if output is not None else None
-    else:
-        process = subprocess.Popen(
-            full_command,
+
+    process = subprocess.Popen(
+        command,
+        stdin=input_file,
+        stdout=None if do_not_capture else (output_file or subprocess.PIPE),
+        stderr=None if do_not_capture else (subprocess.PIPE if output_file else subprocess.STDOUT),
+        shell=shell,
+        env=environment,
+        cwd=working_directory,
+    )
+    if not run_to_completion:
+        return process
+
+    log_outputs(
+        (process,), (input_file, output_file), output_log_level, borg_local_path=borg_local_path
+    )
+
+
+def execute_command_with_processes(
+    full_command,
+    processes,
+    output_log_level=logging.INFO,
+    output_file=None,
+    input_file=None,
+    shell=False,
+    extra_environment=None,
+    working_directory=None,
+    borg_local_path=None,
+):
+    '''
+    Execute the given command (a sequence of command/argument strings) and log its output at the
+    given log level. Simultaneously, continue to poll one or more active processes so that they
+    run as well. This is useful, for instance, for processes that are streaming output to a named
+    pipe that the given command is consuming from.
+
+    If an open output file object is given, then write stdout to the file and only log stderr (but
+    only if an output log level is set). If an open input file object is given, then read stdin from
+    the file.  If shell is True, execute the command within a shell. If an extra environment dict is
+    given, then use it to augment the current environment, and pass the result into the command. If
+    a working directory is given, use that as the present working directory when running the
+    command. If a Borg local path is given, then for any matching command or process (regardless of
+    arguments), treat exit code 1 as a warning instead of an error.
+
+    Raise subprocesses.CalledProcessError if an error occurs while running the command or in the
+    upstream process.
+    '''
+    log_command(full_command, input_file, output_file)
+    environment = {**os.environ, **extra_environment} if extra_environment else None
+    do_not_capture = bool(output_file is DO_NOT_CAPTURE)
+    command = ' '.join(full_command) if shell else full_command
+
+    try:
+        command_process = subprocess.Popen(
+            command,
             stdin=input_file,
-            stdout=output_file or subprocess.PIPE,
-            stderr=subprocess.PIPE if output_file else subprocess.STDOUT,
+            stdout=None if do_not_capture else (output_file or subprocess.PIPE),
+            stderr=None
+            if do_not_capture
+            else (subprocess.PIPE if output_file else subprocess.STDOUT),
             shell=shell,
             env=environment,
             cwd=working_directory,
         )
-        log_output(
-            full_command,
-            process,
-            process.stderr if output_file else process.stdout,
-            output_log_level,
-            error_on_warnings,
-        )
+    except (subprocess.CalledProcessError, OSError):
+        # Something has gone wrong. So vent each process' output buffer to prevent it from hanging.
+        # And then kill the process.
+        for process in processes:
+            if process.poll() is None:
+                process.stdout.read(0)
+                process.kill()
+        raise
 
-
-def execute_command_without_capture(full_command, working_directory=None, error_on_warnings=True):
-    '''
-    Execute the given command (a sequence of command/argument strings), but don't capture or log its
-    output in any way. This is necessary for commands that monkey with the terminal (e.g. progress
-    display) or provide interactive prompts.
-
-    If a working directory is given, use that as the present working directory when running the
-    command. If error on warnings is False, then treat exit code 1 as a warning instead of an error.
-    '''
-    logger.debug(' '.join(full_command))
-
-    try:
-        subprocess.check_call(full_command, cwd=working_directory)
-    except subprocess.CalledProcessError as error:
-        if exit_code_indicates_error(full_command, error.returncode, error_on_warnings):
-            raise
+    log_outputs(
+        tuple(processes) + (command_process,),
+        (input_file, output_file),
+        output_log_level,
+        borg_local_path=borg_local_path,
+    )

borgmatic/hooks/cronhub.py

@@ -13,6 +13,15 @@ MONITOR_STATE_TO_CRONHUB = {
 }
 
 
+def initialize_monitor(
+    ping_url, config_filename, monitoring_log_level, dry_run
+):  # pragma: no cover
+    '''
+    No initialization is necessary for this monitor.
+    '''
+    pass
+
+
 def ping_monitor(ping_url, config_filename, state, monitoring_log_level, dry_run):
     '''
     Ping the given Cronhub URL, modified with the monitor.State. Use the given configuration

borgmatic/hooks/cronitor.py

@@ -13,6 +13,15 @@ MONITOR_STATE_TO_CRONITOR = {
 }
 
 
+def initialize_monitor(
+    ping_url, config_filename, monitoring_log_level, dry_run
+):  # pragma: no cover
+    '''
+    No initialization is necessary for this monitor.
+    '''
+    pass
+
+
 def ping_monitor(ping_url, config_filename, state, monitoring_log_level, dry_run):
     '''
     Ping the given Cronitor URL, modified with the monitor.State. Use the given configuration

borgmatic/hooks/dump.py

@@ -1,4 +1,3 @@
-import glob
 import logging
 import os
 import shutil
@@ -34,64 +33,39 @@ def make_database_dump_filename(dump_path, name, hostname=None):
     return os.path.join(os.path.expanduser(dump_path), hostname or 'localhost', name)
 
 
-def flatten_dump_patterns(dump_patterns, names):
+def create_parent_directory_for_dump(dump_path):
     '''
-    Given a dict from a database hook name to glob patterns matching the dumps for the named
-    databases, flatten out all the glob patterns into a single sequence, and return it.
-
-    Raise ValueError if there are no resulting glob patterns, which indicates that databases are not
-    configured in borgmatic's configuration.
+    Create a directory to contain the given dump path.
     '''
-    flattened = [pattern for patterns in dump_patterns.values() for pattern in patterns]
-
-    if not flattened:
-        raise ValueError(
-            'Cannot restore database(s) {} missing from borgmatic\'s configuration'.format(
-                ', '.join(names) or '"all"'
-            )
-        )
-
-    return flattened
+    os.makedirs(os.path.dirname(dump_path), mode=0o700, exist_ok=True)
 
 
-def remove_database_dumps(dump_path, databases, database_type_name, log_prefix, dry_run):
+def create_named_pipe_for_dump(dump_path):
     '''
-    Remove the database dumps for the given databases in the dump directory path. The databases are
-    supplied as a sequence of dicts, one dict describing each database as per the configuration
-    schema. Use the name of the database type and the log prefix in any log entries. If this is a
-    dry run, then don't actually remove anything.
+    Create a named pipe at the given dump path.
     '''
-    if not databases:
-        logger.debug('{}: No {} databases configured'.format(log_prefix, database_type_name))
-        return
+    create_parent_directory_for_dump(dump_path)
+    os.mkfifo(dump_path, mode=0o600)
 
+
+def remove_database_dumps(dump_path, database_type_name, log_prefix, dry_run):
+    '''
+    Remove all database dumps in the given dump directory path (including the directory itself). If
+    this is a dry run, then don't actually remove anything.
+    '''
     dry_run_label = ' (dry run; not actually removing anything)' if dry_run else ''
 
     logger.info(
         '{}: Removing {} database dumps{}'.format(log_prefix, database_type_name, dry_run_label)
     )
 
-    for database in databases:
-        dump_filename = make_database_dump_filename(
-            dump_path, database['name'], database.get('hostname')
-        )
+    expanded_path = os.path.expanduser(dump_path)
 
-        logger.debug(
-            '{}: Removing {} database dump {} from {}{}'.format(
-                log_prefix, database_type_name, database['name'], dump_filename, dry_run_label
-            )
-        )
-        if dry_run:
-            continue
+    if dry_run:
+        return
 
-        if os.path.isdir(dump_filename):
-            shutil.rmtree(dump_filename)
-        else:
-            os.remove(dump_filename)
-        dump_file_dir = os.path.dirname(dump_filename)
-
-        if len(os.listdir(dump_file_dir)) == 0:
-            os.rmdir(dump_file_dir)
+    if os.path.exists(expanded_path):
+        shutil.rmtree(expanded_path)
 
 
 def convert_glob_patterns_to_borg_patterns(patterns):
@@ -100,80 +74,3 @@ def convert_glob_patterns_to_borg_patterns(patterns):
     patterns like "sh:etc/*".
     '''
     return ['sh:{}'.format(pattern.lstrip(os.path.sep)) for pattern in patterns]
-
-
-def get_database_names_from_dumps(patterns):
-    '''
-    Given a sequence of database dump patterns, find the corresponding database dumps on disk and
-    return the database names from their filenames.
-    '''
-    return [os.path.basename(dump_path) for pattern in patterns for dump_path in glob.glob(pattern)]
-
-
-def get_database_configurations(databases, names):
-    '''
-    Given the full database configuration dicts as per the configuration schema, and a sequence of
-    database names, filter down and yield the configuration for just the named databases.
-    Additionally, if a database configuration is named "all", project out that configuration for
-    each named database.
-    '''
-    named_databases = {database['name']: database for database in databases}
-
-    for name in names:
-        database = named_databases.get(name)
-        if database:
-            yield database
-            continue
-
-        if 'all' in named_databases:
-            yield {**named_databases['all'], **{'name': name}}
-            continue
-
-
-def get_per_hook_database_configurations(hooks, names, dump_patterns):
-    '''
-    Given the hooks configuration dict as per the configuration schema, a sequence of database
-    names to restore, and a dict from database hook name to glob patterns for matching dumps,
-    filter down the configuration for just the named databases.
-
-    If there are no named databases given, then find the corresponding database dumps on disk and
-    use the database names from their filenames. Additionally, if a database configuration is named
-    "all", project out that configuration for each named database.
-
-    Return the results as a dict from database hook name to a sequence of database configuration
-    dicts for that database type.
-
-    Raise ValueError if one of the database names cannot be matched to a database in borgmatic's
-    database configuration.
-    '''
-    hook_databases = {
-        hook_name: list(
-            get_database_configurations(
-                hooks.get(hook_name),
-                names or get_database_names_from_dumps(dump_patterns[hook_name]),
-            )
-        )
-        for hook_name in DATABASE_HOOK_NAMES
-        if hook_name in hooks
-    }
-
-    if not names or 'all' in names:
-        if not any(hook_databases.values()):
-            raise ValueError(
-                'Cannot restore database "all", as there are no database dumps in the archive'
-            )
-
-        return hook_databases
-
-    found_names = {
-        database['name'] for databases in hook_databases.values() for database in databases
-    }
-    missing_names = sorted(set(names) - found_names)
-    if missing_names:
-        raise ValueError(
-            'Cannot restore database(s) {} missing from borgmatic\'s configuration'.format(
-                ', '.join(missing_names)
-            )
-        )
-
-    return hook_databases

borgmatic/hooks/healthchecks.py

@@ -65,20 +65,24 @@ def format_buffered_logs_for_payload():
     return payload
 
 
+def initialize_monitor(
+    ping_url_or_uuid, config_filename, monitoring_log_level, dry_run
+):  # pragma: no cover
+    '''
+    Add a handler to the root logger that stores in memory the most recent logs emitted. That
+    way, we can send them all to Healthchecks upon a finish or failure state.
+    '''
+    logging.getLogger().addHandler(
+        Forgetful_buffering_handler(PAYLOAD_LIMIT_BYTES, monitoring_log_level)
+    )
+
+
 def ping_monitor(ping_url_or_uuid, config_filename, state, monitoring_log_level, dry_run):
     '''
     Ping the given Healthchecks URL or UUID, modified with the monitor.State. Use the given
     configuration filename in any log entries, and log to Healthchecks with the giving log level.
     If this is a dry run, then don't actually ping anything.
     '''
-    if state is monitor.State.START:
-        # Add a handler to the root logger that stores in memory the most recent logs emitted. That
-        # way, we can send them all to Healthchecks upon a finish or failure state.
-        logging.getLogger().addHandler(
-            Forgetful_buffering_handler(PAYLOAD_LIMIT_BYTES, monitoring_log_level)
-        )
-        payload = ''
-
     ping_url = (
         ping_url_or_uuid
         if ping_url_or_uuid.startswith('http')
@@ -97,6 +101,8 @@ def ping_monitor(ping_url_or_uuid, config_filename, state, monitoring_log_level,
 
     if state in (monitor.State.FINISH, monitor.State.FAIL):
         payload = format_buffered_logs_for_payload()
+    else:
+        payload = ''
 
     if not dry_run:
         logging.getLogger('urllib3').setLevel(logging.ERROR)

borgmatic/hooks/mysql.py

@@ -1,7 +1,6 @@
 import logging
-import os
 
-from borgmatic.execute import execute_command
+from borgmatic.execute import execute_command, execute_command_with_processes
 from borgmatic.hooks import dump
 
 logger = logging.getLogger(__name__)
@@ -55,12 +54,16 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run_labe
 
 def dump_databases(databases, log_prefix, location_config, dry_run):
     '''
-    Dump the given MySQL/MariaDB databases to disk. The databases are supplied as a sequence of
-    dicts, one dict describing each database as per the configuration schema. Use the given log
+    Dump the given MySQL/MariaDB databases to a named pipe. The databases are supplied as a sequence
+    of dicts, one dict describing each database as per the configuration schema. Use the given log
     prefix in any log entries. Use the given location configuration dict to construct the
-    destination path. If this is a dry run, then don't actually dump anything.
+    destination path.
+
+    Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named
+    pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence.
     '''
     dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
+    processes = []
 
     logger.info('{}: Dumping MySQL databases{}'.format(log_prefix, dry_run_label))
 
@@ -70,19 +73,25 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
             make_dump_path(location_config), requested_name, database.get('hostname')
         )
         extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
-        dump_command_names = database_names_to_dump(
+        dump_database_names = database_names_to_dump(
             database, extra_environment, log_prefix, dry_run_label
         )
+        if not dump_database_names:
+            raise ValueError('Cannot find any MySQL databases to dump.')
 
         dump_command = (
-            ('mysqldump', '--add-drop-database')
+            ('mysqldump',)
+            + ('--add-drop-database',)
             + (('--host', database['hostname']) if 'hostname' in database else ())
             + (('--port', str(database['port'])) if 'port' in database else ())
             + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
             + (('--user', database['username']) if 'username' in database else ())
             + (tuple(database['options'].split(' ')) if 'options' in database else ())
             + ('--databases',)
-            + dump_command_names
+            + dump_database_names
+            # Use shell redirection rather than execute_command(output_file=open(...)) to prevent
+            # the open() call on a named pipe from hanging the main borgmatic process.
+            + ('>', dump_filename)
         )
 
         logger.debug(
@@ -90,66 +99,77 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
                 log_prefix, requested_name, dump_filename, dry_run_label
             )
         )
-        if not dry_run:
-            os.makedirs(os.path.dirname(dump_filename), mode=0o700, exist_ok=True)
+        if dry_run:
+            continue
+
+        dump.create_named_pipe_for_dump(dump_filename)
+
+        processes.append(
             execute_command(
                 dump_command,
-                output_file=open(dump_filename, 'w'),
+                shell=True,
                 extra_environment=extra_environment,
+                run_to_completion=False,
             )
+        )
+
+    return processes
 
 
 def remove_database_dumps(databases, log_prefix, location_config, dry_run):  # pragma: no cover
     '''
-    Remove the database dumps for the given databases. The databases are supplied as a sequence of
-    dicts, one dict describing each database as per the configuration schema. Use the log prefix in
-    any log entries. Use the given location configuration dict to construct the destination path. If
-    this is a dry run, then don't actually remove anything.
+    Remove all database dump files for this hook regardless of the given databases. Use the log
+    prefix in any log entries. Use the given location configuration dict to construct the
+    destination path. If this is a dry run, then don't actually remove anything.
     '''
-    dump.remove_database_dumps(
-        make_dump_path(location_config), databases, 'MySQL', log_prefix, dry_run
-    )
+    dump.remove_database_dumps(make_dump_path(location_config), 'MySQL', log_prefix, dry_run)
 
 
-def make_database_dump_patterns(databases, log_prefix, location_config, names):
+def make_database_dump_pattern(
+    databases, log_prefix, location_config, name=None
+):  # pragma: no cover
     '''
     Given a sequence of configurations dicts, a prefix to log with, a location configuration dict,
-    and a sequence of database names to match, return the corresponding glob patterns to match the
-    database dumps in an archive. An empty sequence of names indicates that the patterns should
-    match all dumps.
+    and a database name to match, return the corresponding glob patterns to match the database dump
+    in an archive.
     '''
-    return [
-        dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
-        for name in (names or ['*'])
-    ]
+    return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
 
 
-def restore_database_dumps(databases, log_prefix, location_config, dry_run):
+def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
     '''
-    Restore the given MySQL/MariaDB databases from disk. The databases are supplied as a sequence of
-    dicts, one dict describing each database as per the configuration schema. Use the given log
-    prefix in any log entries. Use the given location configuration dict to construct the
-    destination path. If this is a dry run, then don't actually restore anything.
+    Restore the given MySQL/MariaDB database from an extract stream. The database is supplied as a
+    one-element sequence containing a dict describing the database, as per the configuration schema.
+    Use the given log prefix in any log entries. If this is a dry run, then don't actually restore
+    anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce
+    output to consume.
     '''
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
 
-    for database in databases:
-        dump_filename = dump.make_database_dump_filename(
-            make_dump_path(location_config), database['name'], database.get('hostname')
-        )
-        restore_command = (
-            ('mysql', '--batch')
-            + (('--host', database['hostname']) if 'hostname' in database else ())
-            + (('--port', str(database['port'])) if 'port' in database else ())
-            + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
-            + (('--user', database['username']) if 'username' in database else ())
-        )
-        extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
+    if len(database_config) != 1:
+        raise ValueError('The database configuration value is invalid')
 
-        logger.debug(
-            '{}: Restoring MySQL database {}{}'.format(log_prefix, database['name'], dry_run_label)
-        )
-        if not dry_run:
-            execute_command(
-                restore_command, input_file=open(dump_filename), extra_environment=extra_environment
-            )
+    database = database_config[0]
+    restore_command = (
+        ('mysql', '--batch', '--verbose')
+        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--port', str(database['port'])) if 'port' in database else ())
+        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--user', database['username']) if 'username' in database else ())
+    )
+    extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
+
+    logger.debug(
+        '{}: Restoring MySQL database {}{}'.format(log_prefix, database['name'], dry_run_label)
+    )
+    if dry_run:
+        return
+
+    execute_command_with_processes(
+        restore_command,
+        [extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment=extra_environment,
+        borg_local_path=location_config.get('local_path', 'borg'),
+    )

borgmatic/hooks/pagerduty.py

@@ -12,6 +12,15 @@ logger = logging.getLogger(__name__)
 EVENTS_API_URL = 'https://events.pagerduty.com/v2/enqueue'
 
 
+def initialize_monitor(
+    integration_key, config_filename, monitoring_log_level, dry_run
+):  # pragma: no cover
+    '''
+    No initialization is necessary for this monitor.
+    '''
+    pass
+
+
 def ping_monitor(integration_key, config_filename, state, monitoring_log_level, dry_run):
     '''
     If this is an error state, create a PagerDuty event with the given integration key. Use the

borgmatic/hooks/postgresql.py

@@ -1,7 +1,6 @@
 import logging
-import os
 
-from borgmatic.execute import execute_command
+from borgmatic.execute import execute_command, execute_command_with_processes
 from borgmatic.hooks import dump
 
 logger = logging.getLogger(__name__)
@@ -18,12 +17,16 @@ def make_dump_path(location_config):  # pragma: no cover
 
 def dump_databases(databases, log_prefix, location_config, dry_run):
     '''
-    Dump the given PostgreSQL databases to disk. The databases are supplied as a sequence of dicts,
-    one dict describing each database as per the configuration schema. Use the given log prefix in
-    any log entries. Use the given location configuration dict to construct the destination path. If
-    this is a dry run, then don't actually dump anything.
+    Dump the given PostgreSQL databases to a named pipe. The databases are supplied as a sequence of
+    dicts, one dict describing each database as per the configuration schema. Use the given log
+    prefix in any log entries. Use the given location configuration dict to construct the
+    destination path.
+
+    Return a sequence of subprocess.Popen instances for the dump processes ready to spew to a named
+    pipe. But if this is a dry run, then don't actually dump anything and return an empty sequence.
     '''
     dry_run_label = ' (dry run; not actually dumping anything)' if dry_run else ''
+    processes = []
 
     logger.info('{}: Dumping PostgreSQL databases{}'.format(log_prefix, dry_run_label))
 
@@ -33,6 +36,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
             make_dump_path(location_config), name, database.get('hostname')
         )
         all_databases = bool(name == 'all')
+        dump_format = database.get('format', 'custom')
         command = (
             (
                 'pg_dumpall' if all_databases else 'pg_dump',
@@ -40,13 +44,17 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
                 '--clean',
                 '--if-exists',
             )
-            + ('--file', dump_filename)
             + (('--host', database['hostname']) if 'hostname' in database else ())
             + (('--port', str(database['port'])) if 'port' in database else ())
             + (('--username', database['username']) if 'username' in database else ())
-            + (() if all_databases else ('--format', database.get('format', 'custom')))
+            + (() if all_databases else ('--format', dump_format))
+            + (('--file', dump_filename) if dump_format == 'directory' else ())
             + (tuple(database['options'].split(' ')) if 'options' in database else ())
             + (() if all_databases else (name,))
+            # Use shell redirection rather than the --file flag to sidestep synchronization issues
+            # when pg_dump/pg_dumpall tries to write to a named pipe. But for the directory dump
+            # format in a particular, a named destination is required, and redirection doesn't work.
+            + (('>', dump_filename) if dump_format != 'directory' else ())
         )
         extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
 
@@ -55,77 +63,98 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
                 log_prefix, name, dump_filename, dry_run_label
             )
         )
-        if not dry_run:
-            os.makedirs(os.path.dirname(dump_filename), mode=0o700, exist_ok=True)
-            execute_command(command, extra_environment=extra_environment)
+        if dry_run:
+            continue
+
+        if dump_format == 'directory':
+            dump.create_parent_directory_for_dump(dump_filename)
+        else:
+            dump.create_named_pipe_for_dump(dump_filename)
+
+        processes.append(
+            execute_command(
+                command, shell=True, extra_environment=extra_environment, run_to_completion=False
+            )
+        )
+
+    return processes
 
 
 def remove_database_dumps(databases, log_prefix, location_config, dry_run):  # pragma: no cover
     '''
-    Remove the database dumps for the given databases. The databases are supplied as a sequence of
-    dicts, one dict describing each database as per the configuration schema. Use the log prefix in
-    any log entries. Use the given location configuration dict to construct the destination path. If
-    this is a dry run, then don't actually remove anything.
+    Remove all database dump files for this hook regardless of the given databases. Use the log
+    prefix in any log entries. Use the given location configuration dict to construct the
+    destination path. If this is a dry run, then don't actually remove anything.
     '''
-    dump.remove_database_dumps(
-        make_dump_path(location_config), databases, 'PostgreSQL', log_prefix, dry_run
-    )
+    dump.remove_database_dumps(make_dump_path(location_config), 'PostgreSQL', log_prefix, dry_run)
 
 
-def make_database_dump_patterns(databases, log_prefix, location_config, names):
+def make_database_dump_pattern(
+    databases, log_prefix, location_config, name=None
+):  # pragma: no cover
     '''
     Given a sequence of configurations dicts, a prefix to log with, a location configuration dict,
-    and a sequence of database names to match, return the corresponding glob patterns to match the
-    database dumps in an archive. An empty sequence of names indicates that the patterns should
-    match all dumps.
+    and a database name to match, return the corresponding glob patterns to match the database dump
+    in an archive.
     '''
-    return [
-        dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
-        for name in (names or ['*'])
-    ]
+    return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
 
 
-def restore_database_dumps(databases, log_prefix, location_config, dry_run):
+def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
     '''
-    Restore the given PostgreSQL databases from disk. The databases are supplied as a sequence of
-    dicts, one dict describing each database as per the configuration schema. Use the given log
-    prefix in any log entries. Use the given location configuration dict to construct the
-    destination path. If this is a dry run, then don't actually restore anything.
+    Restore the given PostgreSQL database from an extract stream. The database is supplied as a
+    one-element sequence containing a dict describing the database, as per the configuration schema.
+    Use the given log prefix in any log entries. If this is a dry run, then don't actually restore
+    anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce
+    output to consume.
+
+    If the extract process is None, then restore the dump from the filesystem rather than from an
+    extract stream.
     '''
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
 
-    for database in databases:
-        dump_filename = dump.make_database_dump_filename(
-            make_dump_path(location_config), database['name'], database.get('hostname')
-        )
-        all_databases = bool(database['name'] == 'all')
-        analyze_command = (
-            ('psql', '--no-password', '--quiet')
-            + (('--host', database['hostname']) if 'hostname' in database else ())
-            + (('--port', str(database['port'])) if 'port' in database else ())
-            + (('--username', database['username']) if 'username' in database else ())
-            + (('--dbname', database['name']) if not all_databases else ())
-            + ('--command', 'ANALYZE')
-        )
-        restore_command = (
-            ('psql' if all_databases else 'pg_restore', '--no-password')
-            + (
-                ('--if-exists', '--exit-on-error', '--clean', '--dbname', database['name'])
-                if not all_databases
-                else ()
-            )
-            + (('--host', database['hostname']) if 'hostname' in database else ())
-            + (('--port', str(database['port'])) if 'port' in database else ())
-            + (('--username', database['username']) if 'username' in database else ())
-            + (('-f', dump_filename) if all_databases else (dump_filename,))
-        )
-        extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
+    if len(database_config) != 1:
+        raise ValueError('The database configuration value is invalid')
 
-        logger.debug(
-            '{}: Restoring PostgreSQL database {}{}'.format(
-                log_prefix, database['name'], dry_run_label
-            )
+    database = database_config[0]
+    all_databases = bool(database['name'] == 'all')
+    dump_filename = dump.make_database_dump_filename(
+        make_dump_path(location_config), database['name'], database.get('hostname')
+    )
+    analyze_command = (
+        ('psql', '--no-password', '--quiet')
+        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--port', str(database['port'])) if 'port' in database else ())
+        + (('--username', database['username']) if 'username' in database else ())
+        + (('--dbname', database['name']) if not all_databases else ())
+        + ('--command', 'ANALYZE')
+    )
+    restore_command = (
+        ('psql' if all_databases else 'pg_restore', '--no-password')
+        + (
+            ('--if-exists', '--exit-on-error', '--clean', '--dbname', database['name'])
+            if not all_databases
+            else ()
         )
-        if not dry_run:
-            execute_command(restore_command, extra_environment=extra_environment)
-            execute_command(analyze_command, extra_environment=extra_environment)
+        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--port', str(database['port'])) if 'port' in database else ())
+        + (('--username', database['username']) if 'username' in database else ())
+        + (() if extract_process else (dump_filename,))
+    )
+    extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
+
+    logger.debug(
+        '{}: Restoring PostgreSQL database {}{}'.format(log_prefix, database['name'], dry_run_label)
+    )
+    if dry_run:
+        return
+
+    execute_command_with_processes(
+        restore_command,
+        [extract_process] if extract_process else [],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout if extract_process else None,
+        extra_environment=extra_environment,
+        borg_local_path=location_config.get('local_path', 'borg'),
+    )
+    execute_command(analyze_command, extra_environment=extra_environment)

docs/how-to/backup-your-databases.md

@@ -22,13 +22,18 @@ hooks:
         - name: posts
 ```
 
-Prior to each backup, borgmatic dumps each configured database to a file
-and includes it in the backup. After the backup completes, borgmatic removes
-the database dump files to recover disk space.
+As part of each backup, borgmatic streams a database dump for each configured
+database directly to Borg, so it's included in the backup without consuming
+additional disk space. (The one exception is PostgreSQL's "directory" dump
+format, which can't stream and therefore does consume temporary disk space.)
 
-borgmatic creates these temporary dump files in `~/.borgmatic` by default. To
-customize this path, set the `borgmatic_source_directory` option in the
-`location` section of borgmatic's configuration.
+To support this, borgmatic creates temporary named pipes in `~/.borgmatic` by
+default. To customize this path, set the `borgmatic_source_directory` option
+in the `location` section of borgmatic's configuration.
+
+Also note that using a database hook implicitly enables both the
+`read_special` and `one_file_system` configuration settings (even if they're
+disabled in your configuration) to support this dump and restore streaming.
 
 Here's a more involved example that connects to remote databases:
 

docs/how-to/deal-with-very-large-backups.md

@@ -56,6 +56,15 @@ consistency:
         - repository
 ```
 
+Here are the available checks from fastest to slowest:
+
+ * `repository`: Checks the consistency of the repository itself.
+ * `archives`: Checks all of the archives in the repository.
+ * `extract`: Performs an extraction dry-run of the most recent archive.
+ * `data`: Verifies the data integrity of all archives contents, decrypting and decompressing all data (implies `archives` as well).
+
+See [Borg's check documentation](https://borgbackup.readthedocs.io/en/stable/usage/check.html) for more information.
+
 If that's still too slow, you can disable consistency checks entirely,
 either for a single repository or for all repositories.
 

docs/how-to/monitor-your-backups.md

@@ -42,6 +42,7 @@ below for how to configure this.
 software to consume borgmatic JSON output and track when the last
 successful backup occurred. See [scripting
 borgmatic](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#scripting-borgmatic)
+and [related software](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#related-software)
 below for how to configure this.
 5. **Borg hosting providers**: Most [Borg hosting
 providers](https://torsion.org/borgmatic/#hosting-providers) include
@@ -116,21 +117,21 @@ hooks:
 ```
 
 With this hook in place, borgmatic pings your Healthchecks project when a
-backup begins, ends, or errors. Specifically, before the <a
+backup begins, ends, or errors. Specifically, after the <a
 href="https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/">`before_backup`
 hooks</a> run, borgmatic lets Healthchecks know that it has started if any of
 the `prune`, `create`, or `check` actions are run.
 
 Then, if the actions complete successfully, borgmatic notifies Healthchecks of
-the success after the `after_backup` hooks run, and includes borgmatic logs in
+the success before the `after_backup` hooks run, and includes borgmatic logs in
 the payload data sent to Healthchecks. This means that borgmatic logs show up
 in the Healthchecks UI, although be aware that Healthchecks currently has a
 10-kilobyte limit for the logs in each ping.
 
-If an error occurs during any action, borgmatic notifies Healthchecks after
-the `on_error` hooks run, also tacking on logs including the error itself. But
-the logs are only included for errors that occur when a `prune`, `create`, or
-`check` action is run.
+If an error occurs during any action or hook, borgmatic notifies Healthchecks
+before the `on_error` hooks run, also tacking on logs including the error
+itself. But the logs are only included for errors that occur when a `prune`,
+`create`, or `check` action is run.
 
 You can customize the verbosity of the logs that are sent to Healthchecks with
 borgmatic's `--monitoring-verbosity` flag. The `--files` and `--stats` flags
@@ -156,13 +157,13 @@ hooks:
 ```
 
 With this hook in place, borgmatic pings your Cronitor monitor when a backup
-begins, ends, or errors. Specifically, before the <a
+begins, ends, or errors. Specifically, after the <a
 href="https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/">`before_backup`
 hooks</a> run, borgmatic lets Cronitor know that it has started if any of the
 `prune`, `create`, or `check` actions are run. Then, if the actions complete
-successfully, borgmatic notifies Cronitor of the success after the
-`after_backup` hooks run. And if an error occurs during any action, borgmatic
-notifies Cronitor after the `on_error` hooks run.
+successfully, borgmatic notifies Cronitor of the success before the
+`after_backup` hooks run. And if an error occurs during any action or hook,
+borgmatic notifies Cronitor before the `on_error` hooks run.
 
 You can configure Cronitor to notify you by a [variety of
 mechanisms](https://cronitor.io/docs/cron-job-notifications) when backups fail
@@ -184,13 +185,13 @@ hooks:
 ```
 
 With this hook in place, borgmatic pings your Cronhub monitor when a backup
-begins, ends, or errors. Specifically, before the <a
+begins, ends, or errors. Specifically, after the <a
 href="https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/">`before_backup`
 hooks</a> run, borgmatic lets Cronhub know that it has started if any of the
 `prune`, `create`, or `check` actions are run. Then, if the actions complete
-successfully, borgmatic notifies Cronhub of the success after the
-`after_backup` hooks run. And if an error occurs during any action, borgmatic
-notifies Cronhub after the `on_error` hooks run.
+successfully, borgmatic notifies Cronhub of the success before the
+`after_backup` hooks run. And if an error occurs during any action or hook,
+borgmatic notifies Cronhub before the `on_error` hooks run.
 
 Note that even though you configure borgmatic with the "start" variant of the
 ping URL, borgmatic substitutes the correct state into the URL when pinging
@@ -227,7 +228,7 @@ hooks:
 
 With this hook in place, borgmatic creates a PagerDuty event for your service
 whenever backups fail. Specifically, if an error occurs during a `create`,
-`prune`, or `check` action, borgmatic sends an event to PagerDuty after the
+`prune`, or `check` action, borgmatic sends an event to PagerDuty before the
 `on_error` hooks run. Note that borgmatic does not contact PagerDuty when a
 backup starts or ends without error.
 
@@ -250,6 +251,11 @@ suppressed so as not to interfere with the captured JSON. Also note that JSON
 output only shows up at the console, and not in syslog.
 
 
+## Related software
+
+ * [Borgmacator GNOME AppIndicator](https://github.com/N-Coder/borgmacator/)
+
+
 ### Successful backups
 
 `borgmatic list` includes support for a `--successful` flag that only lists

docs/how-to/set-up-backups.md

@@ -12,30 +12,66 @@ First, [install
 Borg](https://borgbackup.readthedocs.io/en/stable/installation.html), at least
 version 1.1.
 
-Then, download and install borgmatic by running the following command:
+Then, download and install borgmatic as a [user site
+installation](https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site)
+by running the following command:
 
 ```bash
 sudo pip3 install --user --upgrade borgmatic
 ```
 
-This is a [recommended user site
-installation](https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site).
-You will need to ensure that `/root/.local/bin` is available on your `$PATH`
-so
-that the borgmatic executable is available. For instance, adding this to
-root's `~/.profile` or `~/.bash_profile` may do the trick:
+This installs borgmatic and its commands at the `/root/.local/bin` path.
+
+Your pip binary may have a different name than "pip3". Make sure you're using
+Python 3, as borgmatic does not support Python 2.
+
+The next step is to ensure that borgmatic's commands available are on your
+system `PATH`, so that you can run borgmatic:
 
 ```bash
-export PATH="$PATH:~/.local/bin"
+echo export 'PATH="$PATH:/root/.local/bin"' >> ~/.bashrc
+source ~/.bashrc
 ```
 
-Note that your pip binary may have a different name than "pip3". Make sure
-you're using Python 3, as borgmatic does not support Python 2.
+This adds `/root/.local/bin` to your non-root user's system `PATH`.
+
+If you're using a command shell other than Bash, you may need to use different
+commands here.
+
+You can check whether all of this worked with:
+
+```bash
+sudo borgmatic --version
+```
+
+If borgmatic is properly installed, that should output your borgmatic version.
+
+
+### Global install option
+
+If you try the user site installation above, and have problems making
+borgmatic commands runnable on your system `PATH`, an alternate approach is to
+install borgmatic globally.
+
+The following uninstalls borgmatic, and then reinstalls it such that borgmatic
+commands are on the default system `PATH`:
+
+```bash
+sudo pip3 uninstall borgmatic
+sudo pip3 install --upgrade borgmatic
+```
+
+The main downside of a global install is that borgmatic is less cleanly
+separated from the rest of your Python software, and there's the theoretical
+possibility of libary conflicts. But if you're okay with that, for instance
+on a relatively dedicated system, then a global install can work out just
+fine.
+
 
 ### Other ways to install
 
-Along with the above process, you have several other options for installing
-borgmatic:
+Besides the approaches described above, there are several other options for
+installing borgmatic:
 
  * [Docker image with scheduled backups](https://hub.docker.com/r/b3vis/borgmatic/)
  * [Docker base image](https://hub.docker.com/r/monachus/borgmatic/)
@@ -170,16 +206,20 @@ good idea to test that borgmatic is working. So to run borgmatic and start a
 backup, you can invoke it like this:
 
 ```bash
-sudo borgmatic --verbosity 1
+sudo borgmatic --verbosity 1 --files
 ```
 
+(No borgmatic `--files` flag? It's only present in newer versions of
+borgmatic. So try leaving it out, or upgrade borgmatic!)
+
 By default, this will also prune any old backups as per the configured
 retention policy, and check backups for consistency problems due to things
 like file damage.
 
-The verbosity flag makes borgmatic list the files that it's archiving, which
-are those that are new or changed since the last backup. Eyeball the list and
-see if it matches your expectations based on the configuration.
+The verbosity flag makes borgmatic show the steps it's performing. And the
+files flag lists each file that's new or changed since the last backup.
+Eyeball the list and see if it matches your expectations based on the
+configuration.
 
 If you'd like to specify an alternate configuration file path, use the
 `--config` flag. See `borgmatic --help` for more information.
@@ -221,6 +261,14 @@ sudo systemctl enable --now borgmatic.timer
 Feel free to modify the timer file based on how frequently you'd like
 borgmatic to run.
 
+### launchd in macOS
+
+If you run borgmatic in macOS with launchd, you may encounter permissions
+issues when reading files to backup. If that happens to you, you may be
+interested in an [unofficial work-around for Full Disk
+Access](https://projects.torsion.org/witten/borgmatic/issues/293).
+
+
 ## Colored output
 
 Borgmatic produces colored terminal output by default. It is disabled when a
@@ -229,6 +277,7 @@ non-interactive terminal is detected (like a cron job), or when you use the
 setting the environment variable `PY_COLORS=False`, or setting the `color`
 option to `false` in the `output` section of configuration.
 
+
 ## Troubleshooting
 
 ### "found character that cannot start any token" error

sample/systemd/borgmatic.service

@@ -19,6 +19,7 @@ Restart=no
 # doesn't support this (pre-240 or so), you may have to remove this option.
 LogRateLimitIntervalSec=0
 
-# Delay start to prevent backups running during boot.
+# Delay start to prevent backups running during boot. Note that systemd-inhibit requires dbus and
+# dbus-user-session to be installed.
 ExecStartPre=sleep 1m
 ExecStart=systemd-inhibit --who="borgmatic" --why="Prevent interrupting scheduled backup" /root/.local/bin/borgmatic --syslog-verbosity 1

scripts/release

@@ -26,7 +26,7 @@ python3 setup.py sdist
 gpg --detach-sign --armor dist/borgmatic-*.tar.gz
 gpg --detach-sign --armor dist/borgmatic-*-py3-none-any.whl
 twine upload -r pypi dist/borgmatic-*.tar.gz dist/borgmatic-*.tar.gz.asc
-twine upload -r pypi dist/borgmatic-*-py3-none-any.whl borgmatic-*-py3-none-any.whl.asc
+twine upload -r pypi dist/borgmatic-*-py3-none-any.whl dist/borgmatic-*-py3-none-any.whl.asc
 
 # Set release changelogs on projects.torsion.org and GitHub.
 release_changelog="$(cat NEWS | sed '/^$/q' | grep -v '^\S')"

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.5.2'
+VERSION = '1.5.6'
 
 
 setup(

tests/end-to-end/test_borgmatic.py

@@ -68,7 +68,7 @@ def test_borgmatic_command():
         extracted_config_path = os.path.join(extract_path, config_path)
         assert open(extracted_config_path).read() == open(config_path).read()
 
-        # Exercise the info flag.
+        # Exercise the info action.
         output = subprocess.check_output(
             'borgmatic --config {} info --json'.format(config_path).split(' ')
         ).decode(sys.stdout.encoding)

tests/end-to-end/test_database.py

@@ -5,12 +5,16 @@ import subprocess
 import sys
 import tempfile
 
+import pytest
 
-def write_configuration(config_path, repository_path, borgmatic_source_directory):
+
+def write_configuration(
+    config_path, repository_path, borgmatic_source_directory, postgresql_dump_format='custom'
+):
     '''
     Write out borgmatic configuration into a file at the config path. Set the options so as to work
     for testing. This includes injecting the given repository path, borgmatic source directory for
-    storing database dumps, and encryption passphrase.
+    storing database dumps, dump format (for PostgreSQL), and encryption passphrase.
     '''
     config = '''
 location:
@@ -29,6 +33,7 @@ hooks:
           hostname: postgresql
           username: postgres
           password: test
+          format: {}
         - name: all
           hostname: postgresql
           username: postgres
@@ -43,7 +48,7 @@ hooks:
           username: root
           password: test
 '''.format(
-        config_path, repository_path, borgmatic_source_directory
+        config_path, repository_path, borgmatic_source_directory, postgresql_dump_format
     )
 
     config_file = open(config_path, 'w')
@@ -67,7 +72,7 @@ def test_database_dump_and_restore():
             'borgmatic -v 2 --config {} init --encryption repokey'.format(config_path).split(' ')
         )
 
-        # Run borgmatic to generate a backup archive including a database dump
+        # Run borgmatic to generate a backup archive including a database dump.
         subprocess.check_call('borgmatic create --config {} -v 2'.format(config_path).split(' '))
 
         # Get the created archive name.
@@ -89,3 +94,71 @@ def test_database_dump_and_restore():
     finally:
         os.chdir(original_working_directory)
         shutil.rmtree(temporary_directory)
+
+
+def test_database_dump_and_restore_with_directory_format():
+    # Create a Borg repository.
+    temporary_directory = tempfile.mkdtemp()
+    repository_path = os.path.join(temporary_directory, 'test.borg')
+    borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic')
+
+    original_working_directory = os.getcwd()
+
+    try:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+        write_configuration(
+            config_path,
+            repository_path,
+            borgmatic_source_directory,
+            postgresql_dump_format='directory',
+        )
+
+        subprocess.check_call(
+            'borgmatic -v 2 --config {} init --encryption repokey'.format(config_path).split(' ')
+        )
+
+        # Run borgmatic to generate a backup archive including a database dump.
+        subprocess.check_call('borgmatic create --config {} -v 2'.format(config_path).split(' '))
+
+        # Restore the database from the archive.
+        subprocess.check_call(
+            'borgmatic --config {} restore --archive latest'.format(config_path).split(' ')
+        )
+    finally:
+        os.chdir(original_working_directory)
+        shutil.rmtree(temporary_directory)
+
+
+def test_database_dump_with_error_causes_borgmatic_to_exit():
+    # Create a Borg repository.
+    temporary_directory = tempfile.mkdtemp()
+    repository_path = os.path.join(temporary_directory, 'test.borg')
+    borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic')
+
+    original_working_directory = os.getcwd()
+
+    try:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+        write_configuration(config_path, repository_path, borgmatic_source_directory)
+
+        subprocess.check_call(
+            'borgmatic -v 2 --config {} init --encryption repokey'.format(config_path).split(' ')
+        )
+
+        # Run borgmatic with a config override such that the database dump fails.
+        with pytest.raises(subprocess.CalledProcessError):
+            subprocess.check_call(
+                [
+                    'borgmatic',
+                    'create',
+                    '--config',
+                    config_path,
+                    '-v',
+                    '2',
+                    '--override',
+                    "hooks.postgresql_databases=[{'name': 'nope'}]",
+                ]
+            )
+    finally:
+        os.chdir(original_working_directory)
+        shutil.rmtree(temporary_directory)

tests/integration/commands/test_arguments.py

@@ -393,12 +393,6 @@ def test_parse_arguments_allows_progress_and_extract():
     module.parse_arguments('--progress', 'extract', '--archive', 'test', 'list')
 
 
-def test_parse_arguments_allows_progress_and_restore():
-    flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
-
-    module.parse_arguments('--progress', 'restore', '--archive', 'test', 'list')
-
-
 def test_parse_arguments_disallows_progress_without_create():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 

tests/integration/test_execute.py

@@ -7,74 +7,139 @@ from flexmock import flexmock
 from borgmatic import execute as module
 
 
-def test_log_output_logs_each_line_separately():
+def test_log_outputs_logs_each_line_separately():
     flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'hi').once()
     flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'there').once()
     flexmock(module).should_receive('exit_code_indicates_error').and_return(False)
 
     hi_process = subprocess.Popen(['echo', 'hi'], stdout=subprocess.PIPE)
-    module.log_output(
-        ['echo', 'hi'],
-        hi_process,
-        hi_process.stdout,
-        output_log_level=logging.INFO,
-        error_on_warnings=False,
-    )
+    flexmock(module).should_receive('output_buffer_for_process').with_args(
+        hi_process, ()
+    ).and_return(hi_process.stdout)
 
     there_process = subprocess.Popen(['echo', 'there'], stdout=subprocess.PIPE)
-    module.log_output(
-        ['echo', 'there'],
-        there_process,
-        there_process.stdout,
+    flexmock(module).should_receive('output_buffer_for_process').with_args(
+        there_process, ()
+    ).and_return(there_process.stdout)
+
+    module.log_outputs(
+        (hi_process, there_process),
+        exclude_stdouts=(),
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
 
-def test_log_output_includes_error_output_in_exception():
+def test_log_outputs_skips_logs_for_process_with_none_stdout():
+    flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'hi').never()
+    flexmock(module.logger).should_receive('log').with_args(logging.INFO, 'there').once()
+    flexmock(module).should_receive('exit_code_indicates_error').and_return(False)
+
+    hi_process = subprocess.Popen(['echo', 'hi'], stdout=None)
+    flexmock(module).should_receive('output_buffer_for_process').with_args(
+        hi_process, ()
+    ).and_return(hi_process.stdout)
+
+    there_process = subprocess.Popen(['echo', 'there'], stdout=subprocess.PIPE)
+    flexmock(module).should_receive('output_buffer_for_process').with_args(
+        there_process, ()
+    ).and_return(there_process.stdout)
+
+    module.log_outputs(
+        (hi_process, there_process),
+        exclude_stdouts=(),
+        output_log_level=logging.INFO,
+        borg_local_path='borg',
+    )
+
+
+def test_log_outputs_includes_error_output_in_exception():
     flexmock(module.logger).should_receive('log')
     flexmock(module).should_receive('exit_code_indicates_error').and_return(True)
+    flexmock(module).should_receive('command_for_process').and_return('grep')
 
     process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout)
 
     with pytest.raises(subprocess.CalledProcessError) as error:
-        module.log_output(
-            ['grep'],
-            process,
-            process.stdout,
+        module.log_outputs(
+            (process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
+        )
+
+    assert error.value.output
+
+
+def test_log_outputs_skips_error_output_in_exception_for_process_with_none_stdout():
+    flexmock(module.logger).should_receive('log')
+    flexmock(module).should_receive('exit_code_indicates_error').and_return(True)
+    flexmock(module).should_receive('command_for_process').and_return('grep')
+
+    process = subprocess.Popen(['grep'], stdout=None)
+    flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout)
+
+    with pytest.raises(subprocess.CalledProcessError) as error:
+        module.log_outputs(
+            (process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
+        )
+
+    assert error.value.returncode == 2
+    assert not error.value.output
+
+
+def test_log_outputs_kills_other_processes_when_one_errors():
+    flexmock(module.logger).should_receive('log')
+    flexmock(module).should_receive('exit_code_indicates_error').and_return(True)
+    flexmock(module).should_receive('command_for_process').and_return('grep')
+
+    process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    other_process = subprocess.Popen(
+        ['watch', 'true'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+    )
+    flexmock(module).should_receive('output_buffer_for_process').with_args(process, ()).and_return(
+        process.stdout
+    )
+    flexmock(module).should_receive('output_buffer_for_process').with_args(
+        other_process, ()
+    ).and_return(other_process.stdout)
+    flexmock(other_process).should_receive('kill').once()
+
+    with pytest.raises(subprocess.CalledProcessError) as error:
+        module.log_outputs(
+            (process, other_process),
+            exclude_stdouts=(),
             output_log_level=logging.INFO,
-            error_on_warnings=False,
+            borg_local_path='borg',
         )
 
     assert error.value.returncode == 2
     assert error.value.output
 
 
-def test_log_output_truncates_long_error_output():
+def test_log_outputs_truncates_long_error_output():
     flexmock(module).ERROR_OUTPUT_MAX_LINE_COUNT = 0
     flexmock(module.logger).should_receive('log')
     flexmock(module).should_receive('exit_code_indicates_error').and_return(True)
+    flexmock(module).should_receive('command_for_process').and_return('grep')
 
     process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout)
 
     with pytest.raises(subprocess.CalledProcessError) as error:
-        module.log_output(
-            ['grep'],
-            process,
-            process.stdout,
-            output_log_level=logging.INFO,
-            error_on_warnings=False,
+        module.log_outputs(
+            (process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
         )
 
     assert error.value.returncode == 2
     assert error.value.output.startswith('...')
 
 
-def test_log_output_with_no_output_logs_nothing():
+def test_log_outputs_with_no_output_logs_nothing():
     flexmock(module.logger).should_receive('log').never()
     flexmock(module).should_receive('exit_code_indicates_error').and_return(False)
 
     process = subprocess.Popen(['true'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    module.log_output(
-        ['true'], process, process.stdout, output_log_level=logging.INFO, error_on_warnings=False
+    flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout)
+
+    module.log_outputs(
+        (process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
     )

tests/unit/borg/test_check.py

@@ -9,9 +9,7 @@ from ..test_verbosity import insert_logging_mock
 
 
 def insert_execute_command_mock(command):
-    flexmock(module).should_receive('execute_command').with_args(
-        command, error_on_warnings=True
-    ).once()
+    flexmock(module).should_receive('execute_command').with_args(command).once()
 
 
 def insert_execute_command_never():
@@ -164,8 +162,8 @@ def test_check_archives_with_progress_calls_borg_with_progress_parameter():
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').and_return(())
     flexmock(module).should_receive('execute_command').never()
-    flexmock(module).should_receive('execute_command_without_capture').with_args(
-        ('borg', 'check', '--progress', 'repo'), error_on_warnings=True
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'check', '--progress', 'repo'), output_file=module.DO_NOT_CAPTURE
     ).once()
 
     module.check_archives(
@@ -179,8 +177,8 @@ def test_check_archives_with_repair_calls_borg_with_repair_parameter():
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').and_return(())
     flexmock(module).should_receive('execute_command').never()
-    flexmock(module).should_receive('execute_command_without_capture').with_args(
-        ('borg', 'check', '--repair', 'repo'), error_on_warnings=True
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'check', '--repair', 'repo'), output_file=module.DO_NOT_CAPTURE
     ).once()
 
     module.check_archives(

tests/unit/borg/test_create.py

@@ -60,6 +60,26 @@ def test_expand_home_directories_considers_none_as_no_directories():
     assert paths == ()
 
 
+@pytest.mark.parametrize(
+    'directories,expected_directories',
+    (
+        (('/', '/root'), ('/',)),
+        (('/', '/root/'), ('/',)),
+        (('/root', '/'), ('/',)),
+        (('/root', '/root/foo'), ('/root',)),
+        (('/root/', '/root/foo'), ('/root/',)),
+        (('/root', '/root/foo/'), ('/root',)),
+        (('/root/foo', '/root'), ('/root',)),
+        (('/root', '/etc', '/root/foo/bar'), ('/etc', '/root')),
+        (('/root', '/root/foo', '/root/foo/bar'), ('/root',)),
+        (('/dup', '/dup'), ('/dup',)),
+        (('/foo', '/bar'), ('/bar', '/foo')),
+    ),
+)
+def test_deduplicate_directories_removes_child_paths(directories, expected_directories):
+    assert module.deduplicate_directories(directories) == expected_directories
+
+
 def test_write_pattern_file_does_not_raise():
     temporary_file = flexmock(name='filename', write=lambda mode: None, flush=lambda: None)
     flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file)
@@ -214,7 +234,8 @@ ARCHIVE_WITH_PATHS = ('repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'bar')
 
 def test_create_archive_calls_borg_with_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -222,7 +243,8 @@ def test_create_archive_calls_borg_with_parameters():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -240,7 +262,8 @@ def test_create_archive_calls_borg_with_parameters():
 def test_create_archive_with_patterns_calls_borg_with_patterns():
     pattern_flags = ('--patterns-from', 'patterns')
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(
         flexmock(name='/tmp/patterns')
@@ -250,7 +273,8 @@ def test_create_archive_with_patterns_calls_borg_with_patterns():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create') + pattern_flags + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -268,7 +292,8 @@ def test_create_archive_with_patterns_calls_borg_with_patterns():
 def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
     exclude_flags = ('--exclude-from', 'excludes')
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(('exclude',))
     flexmock(module).should_receive('_write_pattern_file').and_return(None).and_return(
         flexmock(name='/tmp/excludes')
@@ -278,7 +303,8 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create') + exclude_flags + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -295,7 +321,8 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
 
 def test_create_archive_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -304,7 +331,8 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--info') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.INFO)
 
@@ -322,7 +350,8 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
 
 def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -331,7 +360,8 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--json') + ARCHIVE_WITH_PATHS,
         output_log_level=None,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.INFO)
 
@@ -350,7 +380,8 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
 
 def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -358,7 +389,8 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--debug', '--show-rc') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -376,7 +408,8 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
 
 def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -384,7 +417,8 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--json') + ARCHIVE_WITH_PATHS,
         output_log_level=None,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -403,7 +437,8 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
 
 def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -412,7 +447,8 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--dry-run') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -431,7 +467,8 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete
     # --dry-run and --stats are mutually exclusive, see:
     # https://borgbackup.readthedocs.io/en/stable/usage/create.html#description
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -440,7 +477,8 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--info', '--dry-run') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.INFO)
 
@@ -459,7 +497,8 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete
 
 def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_interval_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -467,7 +506,8 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--checkpoint-interval', '600') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -484,7 +524,8 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
 
 def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -492,7 +533,8 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--chunker-params', '1,2,3,4') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -509,7 +551,8 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param
 
 def test_create_archive_with_compression_calls_borg_with_compression_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -517,7 +560,8 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--compression', 'rle') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -534,7 +578,8 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
 
 def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -542,7 +587,8 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--remote-ratelimit', '100') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -559,7 +605,8 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
 
 def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -567,7 +614,8 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--one-file-system') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -585,7 +633,8 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
 
 def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -593,7 +642,8 @@ def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_paramet
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--numeric-owner') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -611,7 +661,8 @@ def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_paramet
 
 def test_create_archive_with_read_special_calls_borg_with_read_special_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -619,7 +670,8 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--read-special') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -638,7 +690,8 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
 @pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags'))
 def test_create_archive_with_option_true_calls_borg_without_corresponding_parameter(option_name):
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -646,7 +699,8 @@ def test_create_archive_with_option_true_calls_borg_without_corresponding_parame
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -665,7 +719,8 @@ def test_create_archive_with_option_true_calls_borg_without_corresponding_parame
 @pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags'))
 def test_create_archive_with_option_false_calls_borg_with_corresponding_parameter(option_name):
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -673,7 +728,8 @@ def test_create_archive_with_option_false_calls_borg_with_corresponding_paramete
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--no' + option_name.replace('_', '')) + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -691,7 +747,8 @@ def test_create_archive_with_option_false_calls_borg_with_corresponding_paramete
 
 def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -699,7 +756,8 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--files-cache', 'ctime,size') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -717,7 +775,8 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
 
 def test_create_archive_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -725,7 +784,8 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg1', 'create') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg1',
     )
 
     module.create_archive(
@@ -743,7 +803,8 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
 
 def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -751,7 +812,8 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--remote-path', 'borg1') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -769,7 +831,8 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
 
 def test_create_archive_with_umask_calls_borg_with_umask_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -777,7 +840,8 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--umask', '740') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -794,7 +858,8 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
 
 def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -802,7 +867,8 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--lock-wait', '5') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -819,7 +885,8 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
 
 def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_warning_output_log_level():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -827,7 +894,8 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_warning_o
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--stats') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -845,7 +913,8 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_warning_o
 
 def test_create_archive_with_stats_and_log_info_calls_borg_with_stats_parameter_and_info_output_log_level():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -853,7 +922,8 @@ def test_create_archive_with_stats_and_log_info_calls_borg_with_stats_parameter_
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--info', '--stats') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.INFO)
 
@@ -872,7 +942,8 @@ def test_create_archive_with_stats_and_log_info_calls_borg_with_stats_parameter_
 
 def test_create_archive_with_files_calls_borg_with_list_parameter_and_warning_output_log_level():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -880,7 +951,8 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_warning_ou
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--list', '--filter', 'AME-') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -898,7 +970,8 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_warning_ou
 
 def test_create_archive_with_files_and_log_info_calls_borg_with_list_parameter_and_info_output_log_level():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -906,7 +979,8 @@ def test_create_archive_with_files_and_log_info_calls_borg_with_list_parameter_a
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--list', '--filter', 'AME-', '--info') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.INFO)
 
@@ -925,13 +999,17 @@ def test_create_archive_with_files_and_log_info_calls_borg_with_list_parameter_a
 
 def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_parameter_and_no_list():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    flexmock(module).should_receive('execute_command_without_capture').with_args(
-        ('borg', 'create', '--info', '--progress') + ARCHIVE_WITH_PATHS, error_on_warnings=False
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--info', '--progress') + ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
+        output_file=module.DO_NOT_CAPTURE,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.INFO)
 
@@ -950,13 +1028,17 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para
 
 def test_create_archive_with_progress_calls_borg_with_progress_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
-    flexmock(module).should_receive('execute_command_without_capture').with_args(
-        ('borg', 'create', '--progress') + ARCHIVE_WITH_PATHS, error_on_warnings=False
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'create', '--progress') + ARCHIVE_WITH_PATHS,
+        output_log_level=logging.INFO,
+        output_file=module.DO_NOT_CAPTURE,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -972,9 +1054,42 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter():
     )
 
 
+def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progress_parameter():
+    processes = flexmock()
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        ('borg', 'create', '--one-file-system', '--read-special', '--progress')
+        + ARCHIVE_WITH_PATHS,
+        processes=processes,
+        output_log_level=logging.INFO,
+        output_file=module.DO_NOT_CAPTURE,
+        borg_local_path='borg',
+    )
+
+    module.create_archive(
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+        progress=True,
+        stream_processes=processes,
+    )
+
+
 def test_create_archive_with_json_calls_borg_with_json_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -982,7 +1097,8 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--json') + ARCHIVE_WITH_PATHS,
         output_log_level=None,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     ).and_return('[]')
 
     json_output = module.create_archive(
@@ -1002,7 +1118,8 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
 
 def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1010,7 +1127,8 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter()
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--json') + ARCHIVE_WITH_PATHS,
         output_log_level=None,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     ).and_return('[]')
 
     json_output = module.create_archive(
@@ -1031,7 +1149,8 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter()
 
 def test_create_archive_with_source_directories_glob_expands():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'food'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1039,7 +1158,8 @@ def test_create_archive_with_source_directories_glob_expands():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'),
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
 
@@ -1057,7 +1177,8 @@ def test_create_archive_with_source_directories_glob_expands():
 
 def test_create_archive_with_non_matching_source_directories_glob_passes_through():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo*',))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo*',))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1065,7 +1186,8 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo*'),
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([])
 
@@ -1083,7 +1205,8 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
 
 def test_create_archive_with_glob_calls_borg_with_expanded_directories():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'food'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1091,7 +1214,8 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'),
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -1108,7 +1232,8 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
 
 def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1116,7 +1241,8 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', 'repo::ARCHIVE_NAME', 'foo', 'bar'),
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -1133,7 +1259,8 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
 
 def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1141,7 +1268,8 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', 'repo::Documents_{hostname}-{now}', 'foo', 'bar'),
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -1158,7 +1286,8 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
 
 def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1166,7 +1295,8 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'create', '--extra', '--options') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        output_file=None,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -1179,3 +1309,33 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
         },
         storage_config={'extra_borg_options': {'create': '--extra --options'}},
     )
+
+
+def test_create_archive_with_stream_processes_calls_borg_with_processes():
+    processes = flexmock()
+    flexmock(module).should_receive('borgmatic_source_directories').and_return([])
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
+    flexmock(module).should_receive('_expand_home_directories').and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        ('borg', 'create', '--one-file-system', '--read-special') + ARCHIVE_WITH_PATHS,
+        processes=processes,
+        output_log_level=logging.INFO,
+        output_file=None,
+        borg_local_path='borg',
+    )
+
+    module.create_archive(
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={},
+        stream_processes=processes,
+    )

tests/unit/borg/test_environment.py

@@ -60,3 +60,15 @@ def test_initialize_with_relocated_repo_access_should_override_default():
         assert os.environ.get('BORG_RELOCATED_REPO_ACCESS_IS_OK') == 'yes'
     finally:
         os.environ = orig_environ
+
+
+def test_initialize_is_not_affected_by_existing_environment():
+    orig_environ = os.environ
+
+    try:
+        os.environ = {'BORG_PASSPHRASE': 'pass', 'BORG_SSH': 'mosh'}
+        module.initialize({'ssh_command': 'ssh -C'})
+        assert 'BORG_PASSPHRASE' not in os.environ
+        assert os.environ.get('BORG_RSH') == 'ssh -C'
+    finally:
+        os.environ = orig_environ

tests/unit/borg/test_extract.py

@@ -1,5 +1,6 @@
 import logging
 
+import pytest
 from flexmock import flexmock
 
 from borgmatic.borg import extract as module
@@ -7,15 +8,15 @@ from borgmatic.borg import extract as module
 from ..test_verbosity import insert_logging_mock
 
 
-def insert_execute_command_mock(command, working_directory=None, error_on_warnings=True):
+def insert_execute_command_mock(command, working_directory=None):
     flexmock(module).should_receive('execute_command').with_args(
-        command, working_directory=working_directory, error_on_warnings=error_on_warnings
+        command, working_directory=working_directory
     ).once()
 
 
 def insert_execute_command_output_mock(command, result):
     flexmock(module).should_receive('execute_command').with_args(
-        command, output_log_level=None, error_on_warnings=False
+        command, output_log_level=None, borg_local_path=command[0]
     ).and_return(result).once()
 
 
@@ -219,12 +220,27 @@ def test_extract_archive_calls_borg_with_destination_path():
     )
 
 
+def test_extract_archive_calls_borg_with_strip_components():
+    flexmock(module.os.path).should_receive('abspath').and_return('repo')
+    insert_execute_command_mock(('borg', 'extract', '--strip-components', '5', 'repo::archive'))
+
+    module.extract_archive(
+        dry_run=False,
+        repository='repo',
+        archive='archive',
+        paths=None,
+        location_config={},
+        storage_config={},
+        strip_components=5,
+    )
+
+
 def test_extract_archive_calls_borg_with_progress_parameter():
     flexmock(module.os.path).should_receive('abspath').and_return('repo')
-    flexmock(module).should_receive('execute_command_without_capture').with_args(
+    flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'extract', '--progress', 'repo::archive'),
+        output_file=module.DO_NOT_CAPTURE,
         working_directory=None,
-        error_on_warnings=True,
     ).once()
 
     module.extract_archive(
@@ -238,10 +254,50 @@ def test_extract_archive_calls_borg_with_progress_parameter():
     )
 
 
+def test_extract_archive_with_progress_and_extract_to_stdout_raises():
+    flexmock(module).should_receive('execute_command').never()
+
+    with pytest.raises(ValueError):
+        module.extract_archive(
+            dry_run=False,
+            repository='repo',
+            archive='archive',
+            paths=None,
+            location_config={},
+            storage_config={},
+            progress=True,
+            extract_to_stdout=True,
+        )
+
+
+def test_extract_archive_calls_borg_with_stdout_parameter_and_returns_process():
+    flexmock(module.os.path).should_receive('abspath').and_return('repo')
+    process = flexmock()
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'extract', '--stdout', 'repo::archive'),
+        output_file=module.subprocess.PIPE,
+        working_directory=None,
+        run_to_completion=False,
+    ).and_return(process).once()
+
+    assert (
+        module.extract_archive(
+            dry_run=False,
+            repository='repo',
+            archive='archive',
+            paths=None,
+            location_config={},
+            storage_config={},
+            extract_to_stdout=True,
+        )
+        == process
+    )
+
+
 def test_extract_archive_skips_abspath_for_remote_repository():
     flexmock(module.os.path).should_receive('abspath').never()
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'extract', 'server:repo::archive'), working_directory=None, error_on_warnings=True
+        ('borg', 'extract', 'server:repo::archive'), working_directory=None
     ).once()
 
     module.extract_archive(

tests/unit/borg/test_info.py

@@ -10,7 +10,7 @@ from ..test_verbosity import insert_logging_mock
 
 def test_display_archives_info_calls_borg_with_parameters():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', 'repo'), output_log_level=logging.WARNING, error_on_warnings=False
+        ('borg', 'info', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg'
     )
 
     module.display_archives_info(
@@ -20,9 +20,7 @@ def test_display_archives_info_calls_borg_with_parameters():
 
 def test_display_archives_info_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', '--info', 'repo'),
-        output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        ('borg', 'info', '--info', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg'
     )
     insert_logging_mock(logging.INFO)
     module.display_archives_info(
@@ -32,7 +30,7 @@ def test_display_archives_info_with_log_info_calls_borg_with_info_parameter():
 
 def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', '--json', 'repo'), output_log_level=None, error_on_warnings=False
+        ('borg', 'info', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
     ).and_return('[]')
 
     insert_logging_mock(logging.INFO)
@@ -47,7 +45,7 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'info', '--debug', '--show-rc', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -58,7 +56,7 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter():
 
 def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', '--json', 'repo'), output_log_level=None, error_on_warnings=False
+        ('borg', 'info', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
     ).and_return('[]')
 
     insert_logging_mock(logging.DEBUG)
@@ -71,7 +69,7 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp
 
 def test_display_archives_info_with_json_calls_borg_with_json_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', '--json', 'repo'), output_log_level=None, error_on_warnings=False
+        ('borg', 'info', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
     ).and_return('[]')
 
     json_output = module.display_archives_info(
@@ -83,7 +81,7 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter():
 
 def test_display_archives_info_with_archive_calls_borg_with_archive_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', 'repo::archive'), output_log_level=logging.WARNING, error_on_warnings=False
+        ('borg', 'info', 'repo::archive'), output_log_level=logging.WARNING, borg_local_path='borg'
     )
 
     module.display_archives_info(
@@ -93,7 +91,7 @@ def test_display_archives_info_with_archive_calls_borg_with_archive_parameter():
 
 def test_display_archives_info_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg1', 'info', 'repo'), output_log_level=logging.WARNING, error_on_warnings=False
+        ('borg1', 'info', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg1'
     )
 
     module.display_archives_info(
@@ -108,7 +106,7 @@ def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_para
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'info', '--remote-path', 'borg1', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.display_archives_info(
@@ -124,7 +122,7 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'info', '--lock-wait', '5', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.display_archives_info(
@@ -139,7 +137,7 @@ def test_display_archives_info_passes_through_arguments_to_borg(argument_name):
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'info', '--' + argument_name.replace('_', '-'), 'value', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.display_archives_info(

tests/unit/borg/test_init.py

@@ -23,8 +23,8 @@ def insert_info_command_not_found_mock():
 
 
 def insert_init_command_mock(init_command, **kwargs):
-    flexmock(module).should_receive('execute_command_without_capture').with_args(
-        init_command, error_on_warnings=False
+    flexmock(module).should_receive('execute_command').with_args(
+        init_command, output_file=module.DO_NOT_CAPTURE, borg_local_path=init_command[0]
     ).once()
 
 
@@ -37,7 +37,7 @@ def test_initialize_repository_calls_borg_with_parameters():
 
 def test_initialize_repository_raises_for_borg_init_error():
     insert_info_command_not_found_mock()
-    flexmock(module).should_receive('execute_command_without_capture').and_raise(
+    flexmock(module).should_receive('execute_command').and_raise(
         module.subprocess.CalledProcessError(2, 'borg init')
     )
 
@@ -48,8 +48,7 @@ def test_initialize_repository_raises_for_borg_init_error():
 
 
 def test_initialize_repository_skips_initialization_when_repository_already_exists():
-    insert_info_command_found_mock()
-    flexmock(module).should_receive('execute_command_without_capture').never()
+    flexmock(module).should_receive('execute_command').once()
 
     module.initialize_repository(repository='repo', storage_config={}, encryption_mode='repokey')
 

tests/unit/borg/test_list.py

@@ -26,9 +26,7 @@ def test_resolve_archive_name_passes_through_non_latest_archive_name():
 def test_resolve_archive_name_calls_borg_with_parameters():
     expected_archive = 'archive-name'
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
-        output_log_level=None,
-        error_on_warnings=False,
+        ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, output_log_level=None, borg_local_path='borg'
     ).and_return(expected_archive + '\n')
 
     assert module.resolve_archive_name('repo', 'latest', storage_config={}) == expected_archive
@@ -39,7 +37,7 @@ def test_resolve_archive_name_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--info') + BORG_LIST_LATEST_ARGUMENTS,
         output_log_level=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     ).and_return(expected_archive + '\n')
     insert_logging_mock(logging.INFO)
 
@@ -51,7 +49,7 @@ def test_resolve_archive_name_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--debug', '--show-rc') + BORG_LIST_LATEST_ARGUMENTS,
         output_log_level=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     ).and_return(expected_archive + '\n')
     insert_logging_mock(logging.DEBUG)
 
@@ -63,7 +61,7 @@ def test_resolve_archive_name_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg1', 'list') + BORG_LIST_LATEST_ARGUMENTS,
         output_log_level=None,
-        error_on_warnings=False,
+        borg_local_path='borg1',
     ).and_return(expected_archive + '\n')
 
     assert (
@@ -77,7 +75,7 @@ def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_param
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--remote-path', 'borg1') + BORG_LIST_LATEST_ARGUMENTS,
         output_log_level=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     ).and_return(expected_archive + '\n')
 
     assert (
@@ -88,9 +86,7 @@ def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_param
 
 def test_resolve_archive_name_without_archives_raises():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
-        output_log_level=None,
-        error_on_warnings=False,
+        ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, output_log_level=None, borg_local_path='borg'
     ).and_return('')
 
     with pytest.raises(ValueError):
@@ -103,7 +99,7 @@ def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_parameter
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--lock-wait', 'okay') + BORG_LIST_LATEST_ARGUMENTS,
         output_log_level=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     ).and_return(expected_archive + '\n')
 
     assert (
@@ -114,7 +110,7 @@ def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_parameter
 
 def test_list_archives_calls_borg_with_parameters():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', 'repo'), output_log_level=logging.WARNING, error_on_warnings=False
+        ('borg', 'list', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg'
     )
 
     module.list_archives(
@@ -126,9 +122,7 @@ def test_list_archives_calls_borg_with_parameters():
 
 def test_list_archives_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', '--info', 'repo'),
-        output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        ('borg', 'list', '--info', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg'
     )
     insert_logging_mock(logging.INFO)
 
@@ -141,7 +135,7 @@ def test_list_archives_with_log_info_calls_borg_with_info_parameter():
 
 def test_list_archives_with_log_info_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', '--json', 'repo'), output_log_level=None, error_on_warnings=False
+        ('borg', 'list', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
     )
     insert_logging_mock(logging.INFO)
 
@@ -156,7 +150,7 @@ def test_list_archives_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--debug', '--show-rc', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -169,7 +163,7 @@ def test_list_archives_with_log_debug_calls_borg_with_debug_parameter():
 
 def test_list_archives_with_log_debug_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', '--json', 'repo'), output_log_level=None, error_on_warnings=False
+        ('borg', 'list', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -185,7 +179,7 @@ def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--lock-wait', '5', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.list_archives(
@@ -198,7 +192,7 @@ def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
 def test_list_archives_with_archive_calls_borg_with_archive_parameter():
     storage_config = {}
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', 'repo::archive'), output_log_level=logging.WARNING, error_on_warnings=False
+        ('borg', 'list', 'repo::archive'), output_log_level=logging.WARNING, borg_local_path='borg'
     )
 
     module.list_archives(
@@ -213,7 +207,7 @@ def test_list_archives_with_path_calls_borg_with_path_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', 'repo::archive', 'var/lib'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.list_archives(
@@ -225,7 +219,7 @@ def test_list_archives_with_path_calls_borg_with_path_parameter():
 
 def test_list_archives_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg1', 'list', 'repo'), output_log_level=logging.WARNING, error_on_warnings=False
+        ('borg1', 'list', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg1'
     )
 
     module.list_archives(
@@ -240,7 +234,7 @@ def test_list_archives_with_remote_path_calls_borg_with_remote_path_parameters()
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--remote-path', 'borg1', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.list_archives(
@@ -255,7 +249,7 @@ def test_list_archives_with_short_calls_borg_with_short_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--short', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     ).and_return('[]')
 
     module.list_archives(
@@ -283,7 +277,7 @@ def test_list_archives_passes_through_arguments_to_borg(argument_name):
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--' + argument_name.replace('_', '-'), 'value', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     ).and_return('[]')
 
     module.list_archives(
@@ -299,7 +293,7 @@ def test_list_archives_with_successful_calls_borg_to_exclude_checkpoints():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--glob-archives', module.BORG_EXCLUDE_CHECKPOINTS_GLOB, 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     ).and_return('[]')
 
     module.list_archives(
@@ -311,7 +305,7 @@ def test_list_archives_with_successful_calls_borg_to_exclude_checkpoints():
 
 def test_list_archives_with_json_calls_borg_with_json_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', '--json', 'repo'), output_log_level=None, error_on_warnings=False
+        ('borg', 'list', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
     ).and_return('[]')
 
     json_output = module.list_archives(

tests/unit/borg/test_mount.py

@@ -9,7 +9,7 @@ from ..test_verbosity import insert_logging_mock
 
 def insert_execute_command_mock(command):
     flexmock(module).should_receive('execute_command').with_args(
-        command, error_on_warnings=False
+        command, borg_local_path='borg'
     ).once()
 
 
@@ -117,8 +117,10 @@ def test_mount_archive_with_log_debug_calls_borg_with_debug_parameters():
 
 
 def test_mount_archive_calls_borg_with_foreground_parameter():
-    flexmock(module).should_receive('execute_command_without_capture').with_args(
-        ('borg', 'mount', '--foreground', 'repo::archive', '/mnt'), error_on_warnings=False
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'mount', '--foreground', 'repo::archive', '/mnt'),
+        output_file=module.DO_NOT_CAPTURE,
+        borg_local_path='borg',
     ).once()
 
     module.mount_archive(

tests/unit/borg/test_prune.py

@@ -10,7 +10,7 @@ from ..test_verbosity import insert_logging_mock
 
 def insert_execute_command_mock(prune_command, output_log_level):
     flexmock(module).should_receive('execute_command').with_args(
-        prune_command, output_log_level=output_log_level, error_on_warnings=False
+        prune_command, output_log_level=output_log_level, borg_local_path=prune_command[0]
     ).once()
 
 

tests/unit/borg/test_umount.py

@@ -8,9 +8,7 @@ from ..test_verbosity import insert_logging_mock
 
 
 def insert_execute_command_mock(command):
-    flexmock(module).should_receive('execute_command').with_args(
-        command, error_on_warnings=True
-    ).once()
+    flexmock(module).should_receive('execute_command').with_args(command).once()
 
 
 def test_unmount_archive_calls_borg_with_required_parameters():

tests/unit/hooks/test_dump.py

@@ -34,166 +34,42 @@ def test_make_database_dump_filename_with_invalid_name_raises():
         module.make_database_dump_filename('databases', 'invalid/name')
 
 
-def test_flatten_dump_patterns_produces_list_of_all_patterns():
-    dump_patterns = {'postgresql_databases': ['*/glob', 'glob/*'], 'mysql_databases': ['*/*/*']}
-    expected_patterns = sorted(
-        dump_patterns['postgresql_databases'] + dump_patterns['mysql_databases']
-    )
+def test_create_parent_directory_for_dump_does_not_raise():
+    flexmock(module.os).should_receive('makedirs')
 
-    assert sorted(module.flatten_dump_patterns(dump_patterns, ('bob',))) == expected_patterns
+    module.create_parent_directory_for_dump('/path/to/parent')
 
 
-def test_flatten_dump_patterns_with_no_patterns_errors():
-    dump_patterns = {'postgresql_databases': [], 'mysql_databases': []}
+def test_create_named_pipe_for_dump_does_not_raise():
+    flexmock(module).should_receive('create_parent_directory_for_dump')
+    flexmock(module.os).should_receive('mkfifo')
 
-    with pytest.raises(ValueError):
-        assert module.flatten_dump_patterns(dump_patterns, ('bob',))
+    module.create_named_pipe_for_dump('/path/to/pipe')
 
 
-def test_flatten_dump_patterns_with_no_hooks_errors():
-    dump_patterns = {}
+def test_remove_database_dumps_removes_dump_path():
+    flexmock(module.os.path).should_receive('expanduser').and_return('databases/localhost')
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    flexmock(module.shutil).should_receive('rmtree').with_args('databases/localhost').once()
 
-    with pytest.raises(ValueError):
-        assert module.flatten_dump_patterns(dump_patterns, ('bob',))
-
-
-def test_remove_database_dumps_removes_dump_for_each_database():
-    databases = [{'name': 'foo'}, {'name': 'bar'}]
-    flexmock(module).should_receive('make_database_dump_filename').with_args(
-        'databases', 'foo', None
-    ).and_return('databases/localhost/foo')
-    flexmock(module).should_receive('make_database_dump_filename').with_args(
-        'databases', 'bar', None
-    ).and_return('databases/localhost/bar')
-
-    flexmock(module.os.path).should_receive('isdir').and_return(False)
-    flexmock(module.os).should_receive('remove').with_args('databases/localhost/foo').once()
-    flexmock(module.os).should_receive('remove').with_args('databases/localhost/bar').once()
-    flexmock(module.os).should_receive('listdir').with_args('databases/localhost').and_return(
-        ['bar']
-    ).and_return([])
-
-    flexmock(module.os).should_receive('rmdir').with_args('databases/localhost').once()
-
-    module.remove_database_dumps('databases', databases, 'SuperDB', 'test.yaml', dry_run=False)
-
-
-def test_remove_database_dumps_removes_dump_in_directory_format():
-    databases = [{'name': 'foo'}]
-    flexmock(module).should_receive('make_database_dump_filename').with_args(
-        'databases', 'foo', None
-    ).and_return('databases/localhost/foo')
-
-    flexmock(module.os.path).should_receive('isdir').and_return(True)
-    flexmock(module.os).should_receive('remove').never()
-    flexmock(module.shutil).should_receive('rmtree').with_args('databases/localhost/foo').once()
-    flexmock(module.os).should_receive('listdir').with_args('databases/localhost').and_return([])
-    flexmock(module.os).should_receive('rmdir').with_args('databases/localhost').once()
-
-    module.remove_database_dumps('databases', databases, 'SuperDB', 'test.yaml', dry_run=False)
+    module.remove_database_dumps('databases', 'SuperDB', 'test.yaml', dry_run=False)
 
 
 def test_remove_database_dumps_with_dry_run_skips_removal():
-    databases = [{'name': 'foo'}, {'name': 'bar'}]
-    flexmock(module.os).should_receive('rmdir').never()
-    flexmock(module.os).should_receive('remove').never()
+    flexmock(module.os.path).should_receive('expanduser').and_return('databases/localhost')
+    flexmock(module.os.path).should_receive('exists').never()
+    flexmock(module.shutil).should_receive('rmtree').never()
 
-    module.remove_database_dumps('databases', databases, 'SuperDB', 'test.yaml', dry_run=True)
+    module.remove_database_dumps('databases', 'SuperDB', 'test.yaml', dry_run=True)
 
 
-def test_remove_database_dumps_without_databases_does_not_raise():
-    module.remove_database_dumps('databases', [], 'SuperDB', 'test.yaml', dry_run=False)
+def test_remove_database_dumps_without_dump_path_present_skips_removal():
+    flexmock(module.os.path).should_receive('expanduser').and_return('databases/localhost')
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.shutil).should_receive('rmtree').never()
+
+    module.remove_database_dumps('databases', 'SuperDB', 'test.yaml', dry_run=False)
 
 
 def test_convert_glob_patterns_to_borg_patterns_removes_leading_slash():
     assert module.convert_glob_patterns_to_borg_patterns(('/etc/foo/bar',)) == ['sh:etc/foo/bar']
-
-
-def test_get_database_names_from_dumps_gets_names_from_filenames_matching_globs():
-    flexmock(module.glob).should_receive('glob').and_return(
-        ('databases/localhost/foo',)
-    ).and_return(('databases/localhost/bar',)).and_return(())
-
-    assert module.get_database_names_from_dumps(
-        ('databases/*/foo', 'databases/*/bar', 'databases/*/baz')
-    ) == ['foo', 'bar']
-
-
-def test_get_database_configurations_only_produces_named_databases():
-    databases = [
-        {'name': 'foo', 'hostname': 'example.org'},
-        {'name': 'bar', 'hostname': 'example.com'},
-        {'name': 'baz', 'hostname': 'example.org'},
-    ]
-
-    assert list(module.get_database_configurations(databases, ('foo', 'baz'))) == [
-        {'name': 'foo', 'hostname': 'example.org'},
-        {'name': 'baz', 'hostname': 'example.org'},
-    ]
-
-
-def test_get_database_configurations_matches_all_database():
-    databases = [
-        {'name': 'foo', 'hostname': 'example.org'},
-        {'name': 'all', 'hostname': 'example.com'},
-    ]
-
-    assert list(module.get_database_configurations(databases, ('foo', 'bar', 'baz'))) == [
-        {'name': 'foo', 'hostname': 'example.org'},
-        {'name': 'bar', 'hostname': 'example.com'},
-        {'name': 'baz', 'hostname': 'example.com'},
-    ]
-
-
-def test_get_per_hook_database_configurations_partitions_by_hook():
-    hooks = {'postgresql_databases': [flexmock()]}
-    names = ('foo', 'bar')
-    dump_patterns = flexmock()
-    expected_config = {'postgresql_databases': [{'name': 'foo'}, {'name': 'bar'}]}
-    flexmock(module).should_receive('get_database_configurations').with_args(
-        hooks['postgresql_databases'], names
-    ).and_return(expected_config['postgresql_databases'])
-
-    config = module.get_per_hook_database_configurations(hooks, names, dump_patterns)
-
-    assert config == expected_config
-
-
-def test_get_per_hook_database_configurations_defaults_to_detected_database_names():
-    hooks = {'postgresql_databases': [flexmock()]}
-    names = ()
-    detected_names = flexmock()
-    dump_patterns = {'postgresql_databases': [flexmock()]}
-    expected_config = {'postgresql_databases': [flexmock()]}
-    flexmock(module).should_receive('get_database_names_from_dumps').and_return(detected_names)
-    flexmock(module).should_receive('get_database_configurations').with_args(
-        hooks['postgresql_databases'], detected_names
-    ).and_return(expected_config['postgresql_databases'])
-
-    config = module.get_per_hook_database_configurations(hooks, names, dump_patterns)
-
-    assert config == expected_config
-
-
-def test_get_per_hook_database_configurations_with_unknown_database_name_raises():
-    hooks = {'postgresql_databases': [flexmock()]}
-    names = ('foo', 'bar')
-    dump_patterns = flexmock()
-    flexmock(module).should_receive('get_database_configurations').with_args(
-        hooks['postgresql_databases'], names
-    ).and_return([])
-
-    with pytest.raises(ValueError):
-        module.get_per_hook_database_configurations(hooks, names, dump_patterns)
-
-
-def test_get_per_hook_database_configurations_with_all_and_no_archive_dumps_raises():
-    hooks = {'postgresql_databases': [flexmock()]}
-    names = ('foo', 'all')
-    dump_patterns = flexmock()
-    flexmock(module).should_receive('get_database_configurations').with_args(
-        hooks['postgresql_databases'], names
-    ).and_return([])
-
-    with pytest.raises(ValueError):
-        module.get_per_hook_database_configurations(hooks, names, dump_patterns)

tests/unit/hooks/test_mysql.py

@@ -1,5 +1,6 @@
-import sys
+import logging
 
+import pytest
 from flexmock import flexmock
 
 from borgmatic.hooks import mysql as module
@@ -36,7 +37,7 @@ def test_database_names_to_dump_queries_mysql_for_database_names():
 
 def test_dump_databases_runs_mysqldump_for_each_database():
     databases = [{'name': 'foo'}, {'name': 'bar'}]
-    output_file = flexmock()
+    processes = [flexmock(), flexmock()]
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
@@ -44,17 +45,24 @@ def test_dump_databases_runs_mysqldump_for_each_database():
     flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
         ('bar',)
     )
-    flexmock(module.os).should_receive('makedirs')
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(output_file)
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
 
-    for name in ('foo', 'bar'):
+    for name, process in zip(('foo', 'bar'), processes):
         flexmock(module).should_receive('execute_command').with_args(
-            ('mysqldump', '--add-drop-database', '--databases', name),
-            output_file=output_file,
+            (
+                'mysqldump',
+                '--add-drop-database',
+                '--databases',
+                name,
+                '>',
+                'databases/localhost/{}'.format(name),
+            ),
+            shell=True,
             extra_environment=None,
-        ).once()
+            run_to_completion=False,
+        ).and_return(process).once()
 
-    module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == processes
 
 
 def test_dump_databases_with_dry_run_skips_mysqldump():
@@ -66,7 +74,7 @@ def test_dump_databases_with_dry_run_skips_mysqldump():
     flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
         ('bar',)
     )
-    flexmock(module.os).should_receive('makedirs').never()
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
     flexmock(module).should_receive('execute_command').never()
 
     module.dump_databases(databases, 'test.yaml', {}, dry_run=True)
@@ -74,14 +82,13 @@ def test_dump_databases_with_dry_run_skips_mysqldump():
 
 def test_dump_databases_runs_mysqldump_with_hostname_and_port():
     databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
-    output_file = flexmock()
+    process = flexmock()
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/database.example.org/foo'
     )
     flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
-    flexmock(module.os).should_receive('makedirs')
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(output_file)
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
 
     flexmock(module).should_receive('execute_command').with_args(
         (
@@ -95,131 +102,154 @@ def test_dump_databases_runs_mysqldump_with_hostname_and_port():
             'tcp',
             '--databases',
             'foo',
+            '>',
+            'databases/database.example.org/foo',
         ),
-        output_file=output_file,
+        shell=True,
         extra_environment=None,
-    ).once()
+        run_to_completion=False,
+    ).and_return(process).once()
 
-    module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
 
 
 def test_dump_databases_runs_mysqldump_with_username_and_password():
     databases = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}]
-    output_file = flexmock()
+    process = flexmock()
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     )
     flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
-    flexmock(module.os).should_receive('makedirs')
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(output_file)
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
 
     flexmock(module).should_receive('execute_command').with_args(
-        ('mysqldump', '--add-drop-database', '--user', 'root', '--databases', 'foo'),
-        output_file=output_file,
+        (
+            'mysqldump',
+            '--add-drop-database',
+            '--user',
+            'root',
+            '--databases',
+            'foo',
+            '>',
+            'databases/localhost/foo',
+        ),
+        shell=True,
         extra_environment={'MYSQL_PWD': 'trustsome1'},
-    ).once()
+        run_to_completion=False,
+    ).and_return(process).once()
 
-    module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
 
 
 def test_dump_databases_runs_mysqldump_with_options():
     databases = [{'name': 'foo', 'options': '--stuff=such'}]
-    output_file = flexmock()
+    process = flexmock()
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     )
     flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
-    flexmock(module.os).should_receive('makedirs')
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(output_file)
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
 
     flexmock(module).should_receive('execute_command').with_args(
-        ('mysqldump', '--add-drop-database', '--stuff=such', '--databases', 'foo'),
-        output_file=output_file,
+        (
+            'mysqldump',
+            '--add-drop-database',
+            '--stuff=such',
+            '--databases',
+            'foo',
+            '>',
+            'databases/localhost/foo',
+        ),
+        shell=True,
         extra_environment=None,
-    ).once()
+        run_to_completion=False,
+    ).and_return(process).once()
 
-    module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
 
 
 def test_dump_databases_runs_mysqldump_for_all_databases():
     databases = [{'name': 'all'}]
-    output_file = flexmock()
+    process = flexmock()
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/all'
     )
     flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar'))
-    flexmock(module.os).should_receive('makedirs')
-    flexmock(sys.modules['builtins']).should_receive('open').and_return(output_file)
-
-    flexmock(module).should_receive('execute_command').with_args(
-        ('mysqldump', '--add-drop-database', '--databases', 'foo', 'bar'),
-        output_file=output_file,
-        extra_environment=None,
-    ).once()
-
-    module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
-
-
-def test_make_database_dump_patterns_converts_names_to_glob_paths():
-    flexmock(module).should_receive('make_dump_path').and_return('')
-    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
-        'databases/*/foo'
-    ).and_return('databases/*/bar')
-
-    assert module.make_database_dump_patterns(flexmock(), flexmock(), {}, ('foo', 'bar')) == [
-        'databases/*/foo',
-        'databases/*/bar',
-    ]
-
-
-def test_make_database_dump_patterns_treats_empty_names_as_matching_all_databases():
-    flexmock(module).should_receive('make_dump_path').and_return('/dump/path')
-    flexmock(module.dump).should_receive('make_database_dump_filename').with_args(
-        '/dump/path', '*', '*'
-    ).and_return('databases/*/*')
-
-    assert module.make_database_dump_patterns(flexmock(), flexmock(), {}, ()) == ['databases/*/*']
-
-
-def test_restore_database_dumps_restores_each_database():
-    databases = [{'name': 'foo'}, {'name': 'bar'}]
-    flexmock(module).should_receive('make_dump_path').and_return('')
-    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
-        'databases/localhost/foo'
-    ).and_return('databases/localhost/bar')
-
-    for name in ('foo', 'bar'):
-        dump_filename = 'databases/localhost/{}'.format(name)
-        input_file = flexmock()
-        flexmock(sys.modules['builtins']).should_receive('open').with_args(
-            dump_filename
-        ).and_return(input_file)
-        flexmock(module).should_receive('execute_command').with_args(
-            ('mysql', '--batch'), input_file=input_file, extra_environment=None
-        ).once()
-
-    module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)
-
-
-def test_restore_database_dumps_runs_mysql_with_hostname_and_port():
-    databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
-    flexmock(module).should_receive('make_dump_path').and_return('')
-    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
-        'databases/localhost/foo'
-    )
-    dump_filename = 'databases/localhost/foo'
-    input_file = flexmock()
-    flexmock(sys.modules['builtins']).should_receive('open').with_args(dump_filename).and_return(
-        input_file
-    )
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
 
     flexmock(module).should_receive('execute_command').with_args(
+        (
+            'mysqldump',
+            '--add-drop-database',
+            '--databases',
+            'foo',
+            'bar',
+            '>',
+            'databases/localhost/all',
+        ),
+        shell=True,
+        extra_environment=None,
+        run_to_completion=False,
+    ).and_return(process).once()
+
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
+
+
+def test_dump_databases_errors_for_missing_all_databases():
+    databases = [{'name': 'all'}]
+    process = flexmock()
+    flexmock(module).should_receive('make_dump_path').and_return('')
+    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
+        'databases/localhost/all'
+    )
+    flexmock(module).should_receive('database_names_to_dump').and_return(())
+
+    with pytest.raises(ValueError):
+        assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
+
+
+def test_restore_database_dump_runs_mysql_to_restore():
+    database_config = [{'name': 'foo'}]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        ('mysql', '--batch', '--verbose'),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment=None,
+        borg_local_path='borg',
+    ).once()
+
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+    )
+
+
+def test_restore_database_dump_errors_on_multiple_database_config():
+    database_config = [{'name': 'foo'}, {'name': 'bar'}]
+
+    flexmock(module).should_receive('execute_command_with_processes').never()
+    flexmock(module).should_receive('execute_command').never()
+
+    with pytest.raises(ValueError):
+        module.restore_database_dump(
+            database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock()
+        )
+
+
+def test_restore_database_dump_runs_mysql_with_hostname_and_port():
+    database_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
         (
             'mysql',
             '--batch',
+            '--verbose',
             '--host',
             'database.example.org',
             '--port',
@@ -227,29 +257,41 @@ def test_restore_database_dumps_runs_mysql_with_hostname_and_port():
             '--protocol',
             'tcp',
         ),
-        input_file=input_file,
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
         extra_environment=None,
+        borg_local_path='borg',
     ).once()
 
-    module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)
-
-
-def test_restore_database_dumps_runs_mysql_with_username_and_password():
-    databases = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}]
-    flexmock(module).should_receive('make_dump_path').and_return('')
-    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
-        'databases/localhost/foo'
-    )
-    dump_filename = 'databases/localhost/foo'
-    input_file = flexmock()
-    flexmock(sys.modules['builtins']).should_receive('open').with_args(dump_filename).and_return(
-        input_file
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
     )
 
-    flexmock(module).should_receive('execute_command').with_args(
-        ('mysql', '--batch', '--user', 'root'),
-        input_file=input_file,
+
+def test_restore_database_dump_runs_mysql_with_username_and_password():
+    database_config = [{'name': 'foo', 'username': 'root', 'password': 'trustsome1'}]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        ('mysql', '--batch', '--verbose', '--user', 'root'),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
         extra_environment={'MYSQL_PWD': 'trustsome1'},
+        borg_local_path='borg',
     ).once()
 
-    module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+    )
+
+
+def test_restore_database_dump_with_dry_run_skips_restore():
+    database_config = [{'name': 'foo'}]
+
+    flexmock(module).should_receive('execute_command_with_processes').never()
+
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock()
+    )

tests/unit/hooks/test_postgresql.py

@@ -1,3 +1,6 @@
+import logging
+
+import pytest
 from flexmock import flexmock
 
 from borgmatic.hooks import postgresql as module
@@ -5,29 +8,32 @@ from borgmatic.hooks import postgresql as module
 
 def test_dump_databases_runs_pg_dump_for_each_database():
     databases = [{'name': 'foo'}, {'name': 'bar'}]
+    processes = [flexmock(), flexmock()]
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     ).and_return('databases/localhost/bar')
-    flexmock(module.os).should_receive('makedirs')
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
 
-    for name in ('foo', 'bar'):
+    for name, process in zip(('foo', 'bar'), processes):
         flexmock(module).should_receive('execute_command').with_args(
             (
                 'pg_dump',
                 '--no-password',
                 '--clean',
                 '--if-exists',
-                '--file',
-                'databases/localhost/{}'.format(name),
                 '--format',
                 'custom',
                 name,
+                '>',
+                'databases/localhost/{}'.format(name),
             ),
+            shell=True,
             extra_environment=None,
-        ).once()
+            run_to_completion=False,
+        ).and_return(process).once()
 
-    module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == processes
 
 
 def test_dump_databases_with_dry_run_skips_pg_dump():
@@ -36,19 +42,20 @@ def test_dump_databases_with_dry_run_skips_pg_dump():
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     ).and_return('databases/localhost/bar')
-    flexmock(module.os).should_receive('makedirs').never()
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
     flexmock(module).should_receive('execute_command').never()
 
-    module.dump_databases(databases, 'test.yaml', {}, dry_run=True)
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=True) == []
 
 
 def test_dump_databases_runs_pg_dump_with_hostname_and_port():
     databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
+    process = flexmock()
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/database.example.org/foo'
     )
-    flexmock(module.os).should_receive('makedirs')
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
 
     flexmock(module).should_receive('execute_command').with_args(
         (
@@ -56,8 +63,6 @@ def test_dump_databases_runs_pg_dump_with_hostname_and_port():
             '--no-password',
             '--clean',
             '--if-exists',
-            '--file',
-            'databases/database.example.org/foo',
             '--host',
             'database.example.org',
             '--port',
@@ -65,20 +70,25 @@ def test_dump_databases_runs_pg_dump_with_hostname_and_port():
             '--format',
             'custom',
             'foo',
+            '>',
+            'databases/database.example.org/foo',
         ),
+        shell=True,
         extra_environment=None,
-    ).once()
+        run_to_completion=False,
+    ).and_return(process).once()
 
-    module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
 
 
 def test_dump_databases_runs_pg_dump_with_username_and_password():
     databases = [{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1'}]
+    process = flexmock()
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     )
-    flexmock(module.os).should_receive('makedirs')
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
 
     flexmock(module).should_receive('execute_command').with_args(
         (
@@ -86,27 +96,31 @@ def test_dump_databases_runs_pg_dump_with_username_and_password():
             '--no-password',
             '--clean',
             '--if-exists',
-            '--file',
-            'databases/localhost/foo',
             '--username',
             'postgres',
             '--format',
             'custom',
             'foo',
+            '>',
+            'databases/localhost/foo',
         ),
+        shell=True,
         extra_environment={'PGPASSWORD': 'trustsome1'},
-    ).once()
+        run_to_completion=False,
+    ).and_return(process).once()
 
-    module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
 
 
-def test_dump_databases_runs_pg_dump_with_format():
-    databases = [{'name': 'foo', 'format': 'tar'}]
+def test_dump_databases_runs_pg_dump_with_directory_format():
+    databases = [{'name': 'foo', 'format': 'directory'}]
+    process = flexmock()
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     )
-    flexmock(module.os).should_receive('makedirs')
+    flexmock(module.dump).should_receive('create_parent_directory_for_dump')
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
 
     flexmock(module).should_receive('execute_command').with_args(
         (
@@ -114,25 +128,28 @@ def test_dump_databases_runs_pg_dump_with_format():
             '--no-password',
             '--clean',
             '--if-exists',
+            '--format',
+            'directory',
             '--file',
             'databases/localhost/foo',
-            '--format',
-            'tar',
             'foo',
         ),
+        shell=True,
         extra_environment=None,
-    ).once()
+        run_to_completion=False,
+    ).and_return(process).once()
 
-    module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
 
 
 def test_dump_databases_runs_pg_dump_with_options():
     databases = [{'name': 'foo', 'options': '--stuff=such'}]
+    process = flexmock()
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     )
-    flexmock(module.os).should_receive('makedirs')
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
 
     flexmock(module).should_receive('execute_command').with_args(
         (
@@ -140,100 +157,93 @@ def test_dump_databases_runs_pg_dump_with_options():
             '--no-password',
             '--clean',
             '--if-exists',
-            '--file',
-            'databases/localhost/foo',
             '--format',
             'custom',
             '--stuff=such',
             'foo',
+            '>',
+            'databases/localhost/foo',
         ),
+        shell=True,
         extra_environment=None,
-    ).once()
+        run_to_completion=False,
+    ).and_return(process).once()
 
-    module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
 
 
 def test_dump_databases_runs_pg_dumpall_for_all_databases():
     databases = [{'name': 'all'}]
+    process = flexmock()
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/all'
     )
-    flexmock(module.os).should_receive('makedirs')
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
 
     flexmock(module).should_receive('execute_command').with_args(
+        ('pg_dumpall', '--no-password', '--clean', '--if-exists', '>', 'databases/localhost/all'),
+        shell=True,
+        extra_environment=None,
+        run_to_completion=False,
+    ).and_return(process).once()
+
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
+
+
+def test_restore_database_dump_runs_pg_restore():
+    database_config = [{'name': 'foo'}]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
         (
-            'pg_dumpall',
+            'pg_restore',
             '--no-password',
-            '--clean',
             '--if-exists',
-            '--file',
-            'databases/localhost/all',
+            '--exit-on-error',
+            '--clean',
+            '--dbname',
+            'foo',
         ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment=None,
+        borg_local_path='borg',
+    ).once()
+    flexmock(module).should_receive('execute_command').with_args(
+        ('psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'),
         extra_environment=None,
     ).once()
 
-    module.dump_databases(databases, 'test.yaml', {}, dry_run=False)
-
-
-def test_make_database_dump_patterns_converts_names_to_glob_paths():
-    flexmock(module).should_receive('make_dump_path').and_return('')
-    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
-        'databases/*/foo'
-    ).and_return('databases/*/bar')
-
-    assert module.make_database_dump_patterns(flexmock(), flexmock(), {}, ('foo', 'bar')) == [
-        'databases/*/foo',
-        'databases/*/bar',
-    ]
-
-
-def test_make_database_dump_patterns_treats_empty_names_as_matching_all_databases():
-    flexmock(module).should_receive('make_dump_path').and_return('/dump/path')
-    flexmock(module.dump).should_receive('make_database_dump_filename').with_args(
-        '/dump/path', '*', '*'
-    ).and_return('databases/*/*')
-
-    assert module.make_database_dump_patterns(flexmock(), flexmock(), {}, ()) == ['databases/*/*']
-
-
-def test_restore_database_dumps_restores_each_database():
-    databases = [{'name': 'foo'}, {'name': 'bar'}]
-    flexmock(module).should_receive('make_dump_path').and_return('')
-    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
-        'databases/localhost/foo'
-    ).and_return('databases/localhost/bar')
-
-    for name in ('foo', 'bar'):
-        flexmock(module).should_receive('execute_command').with_args(
-            (
-                'pg_restore',
-                '--no-password',
-                '--if-exists',
-                '--exit-on-error',
-                '--clean',
-                '--dbname',
-                name,
-                'databases/localhost/{}'.format(name),
-            ),
-            extra_environment=None,
-        ).once()
-        flexmock(module).should_receive('execute_command').with_args(
-            ('psql', '--no-password', '--quiet', '--dbname', name, '--command', 'ANALYZE'),
-            extra_environment=None,
-        ).once()
-
-    module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)
-
-
-def test_restore_database_dumps_runs_pg_restore_with_hostname_and_port():
-    databases = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
-    flexmock(module).should_receive('make_dump_path').and_return('')
-    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
-        'databases/localhost/foo'
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
     )
 
-    flexmock(module).should_receive('execute_command').with_args(
+
+def test_restore_database_dump_errors_on_multiple_database_config():
+    database_config = [{'name': 'foo'}, {'name': 'bar'}]
+
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
+    flexmock(module).should_receive('execute_command_with_processes').never()
+    flexmock(module).should_receive('execute_command').never()
+
+    with pytest.raises(ValueError):
+        module.restore_database_dump(
+            database_config, 'test.yaml', {}, dry_run=False, extract_process=flexmock()
+        )
+
+
+def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
+    database_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
         (
             'pg_restore',
             '--no-password',
@@ -246,9 +256,12 @@ def test_restore_database_dumps_runs_pg_restore_with_hostname_and_port():
             'database.example.org',
             '--port',
             '5433',
-            'databases/localhost/foo',
         ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
         extra_environment=None,
+        borg_local_path='borg',
     ).once()
     flexmock(module).should_receive('execute_command').with_args(
         (
@@ -267,17 +280,18 @@ def test_restore_database_dumps_runs_pg_restore_with_hostname_and_port():
         extra_environment=None,
     ).once()
 
-    module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)
-
-
-def test_restore_database_dumps_runs_pg_restore_with_username_and_password():
-    databases = [{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1'}]
-    flexmock(module).should_receive('make_dump_path').and_return('')
-    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
-        'databases/localhost/foo'
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
     )
 
-    flexmock(module).should_receive('execute_command').with_args(
+
+def test_restore_database_dump_runs_pg_restore_with_username_and_password():
+    database_config = [{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1'}]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
         (
             'pg_restore',
             '--no-password',
@@ -288,9 +302,12 @@ def test_restore_database_dumps_runs_pg_restore_with_username_and_password():
             'foo',
             '--username',
             'postgres',
-            'databases/localhost/foo',
         ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
         extra_environment={'PGPASSWORD': 'trustsome1'},
+        borg_local_path='borg',
     ).once()
     flexmock(module).should_receive('execute_command').with_args(
         (
@@ -307,21 +324,73 @@ def test_restore_database_dumps_runs_pg_restore_with_username_and_password():
         extra_environment={'PGPASSWORD': 'trustsome1'},
     ).once()
 
-    module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)
-
-
-def test_restore_database_dumps_runs_psql_for_all_database_dump():
-    databases = [{'name': 'all'}]
-    flexmock(module).should_receive('make_dump_path').and_return('')
-    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
-        'databases/localhost/all'
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
     )
 
-    flexmock(module).should_receive('execute_command').with_args(
-        ('psql', '--no-password', '-f', 'databases/localhost/all'), extra_environment=None
+
+def test_restore_database_dump_runs_psql_for_all_database_dump():
+    database_config = [{'name': 'all'}]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        ('psql', '--no-password'),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment=None,
+        borg_local_path='borg',
     ).once()
     flexmock(module).should_receive('execute_command').with_args(
         ('psql', '--no-password', '--quiet', '--command', 'ANALYZE'), extra_environment=None
     ).once()
 
-    module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+    )
+
+
+def test_restore_database_dump_with_dry_run_skips_restore():
+    database_config = [{'name': 'foo'}]
+
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
+    flexmock(module).should_receive('execute_command_with_processes').never()
+
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock()
+    )
+
+
+def test_restore_database_dump_without_extract_process_restores_from_disk():
+    database_config = [{'name': 'foo'}]
+
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename').and_return('/dump/path')
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        (
+            'pg_restore',
+            '--no-password',
+            '--if-exists',
+            '--exit-on-error',
+            '--clean',
+            '--dbname',
+            'foo',
+            '/dump/path',
+        ),
+        processes=[],
+        output_log_level=logging.DEBUG,
+        input_file=None,
+        extra_environment=None,
+        borg_local_path='borg',
+    ).once()
+    flexmock(module).should_receive('execute_command').with_args(
+        ('psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'),
+        extra_environment=None,
+    ).once()
+
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=False, extract_process=None
+    )

tests/unit/test_execute.py

@@ -1,3 +1,5 @@
+import subprocess
+
 import pytest
 from flexmock import flexmock
 
@@ -5,24 +7,56 @@ from borgmatic import execute as module
 
 
 @pytest.mark.parametrize(
-    'exit_code,error_on_warnings,expected_result',
+    'process,exit_code,borg_local_path,expected_result',
     (
-        (2, True, True),
-        (2, False, True),
-        (1, True, True),
-        (1, False, False),
-        (0, True, False),
-        (0, False, False),
+        (flexmock(args=['grep']), 2, None, True),
+        (flexmock(args=['grep']), 2, 'borg', True),
+        (flexmock(args=['borg']), 2, 'borg', True),
+        (flexmock(args=['borg1']), 2, 'borg1', True),
+        (flexmock(args=['grep']), 1, None, True),
+        (flexmock(args=['grep']), 1, 'borg', True),
+        (flexmock(args=['borg']), 1, 'borg', False),
+        (flexmock(args=['borg1']), 1, 'borg1', False),
+        (flexmock(args=['grep']), 0, None, False),
+        (flexmock(args=['grep']), 0, 'borg', False),
+        (flexmock(args=['borg']), 0, 'borg', False),
+        (flexmock(args=['borg1']), 0, 'borg1', False),
+        (flexmock(args=['borg']), None, None, False),
     ),
 )
-def test_exit_code_indicates_error_respects_exit_code_and_error_on_warnings(
-    exit_code, error_on_warnings, expected_result
+def test_exit_code_indicates_error_respects_exit_code_and_borg_local_path(
+    process, exit_code, borg_local_path, expected_result
 ):
+    assert module.exit_code_indicates_error(process, exit_code, borg_local_path) is expected_result
+
+
+def test_command_for_process_converts_sequence_command_to_string():
+    process = flexmock(args=['foo', 'bar', 'baz'])
+
+    assert module.command_for_process(process) == 'foo bar baz'
+
+
+def test_command_for_process_passes_through_string_command():
+    process = flexmock(args='foo bar baz')
+
+    assert module.command_for_process(process) == 'foo bar baz'
+
+
+def test_output_buffer_for_process_returns_stderr_when_stdout_excluded():
+    stdout = flexmock()
+    stderr = flexmock()
+    process = flexmock(stdout=stdout, stderr=stderr)
+
+    assert module.output_buffer_for_process(process, exclude_stdouts=[flexmock(), stdout]) == stderr
+
+
+def test_output_buffer_for_process_returns_stdout_when_not_excluded():
+    stdout = flexmock()
+    process = flexmock(stdout=stdout)
+
     assert (
-        module.exit_code_indicates_error(
-            ('command',), exit_code, error_on_warnings=error_on_warnings
-        )
-        is expected_result
+        module.output_buffer_for_process(process, exclude_stdouts=[flexmock(), flexmock()])
+        == stdout
     )
 
 
@@ -38,7 +72,7 @@ def test_execute_command_calls_full_command():
         env=None,
         cwd=None,
     ).and_return(flexmock(stdout=None)).once()
-    flexmock(module).should_receive('log_output')
+    flexmock(module).should_receive('log_outputs')
 
     output = module.execute_command(full_command)
 
@@ -58,13 +92,27 @@ def test_execute_command_calls_full_command_with_output_file():
         env=None,
         cwd=None,
     ).and_return(flexmock(stderr=None)).once()
-    flexmock(module).should_receive('log_output')
+    flexmock(module).should_receive('log_outputs')
 
     output = module.execute_command(full_command, output_file=output_file)
 
     assert output is None
 
 
+def test_execute_command_calls_full_command_without_capturing_output():
+    full_command = ['foo', 'bar']
+    flexmock(module.os, environ={'a': 'b'})
+    flexmock(module.subprocess).should_receive('Popen').with_args(
+        full_command, stdin=None, stdout=None, stderr=None, shell=False, env=None, cwd=None
+    ).and_return(flexmock(wait=lambda: 0)).once()
+    flexmock(module).should_receive('exit_code_indicates_error').and_return(False)
+    flexmock(module).should_receive('log_outputs')
+
+    output = module.execute_command(full_command, output_file=module.DO_NOT_CAPTURE)
+
+    assert output is None
+
+
 def test_execute_command_calls_full_command_with_input_file():
     full_command = ['foo', 'bar']
     input_file = flexmock(name='test')
@@ -78,7 +126,7 @@ def test_execute_command_calls_full_command_with_input_file():
         env=None,
         cwd=None,
     ).and_return(flexmock(stdout=None)).once()
-    flexmock(module).should_receive('log_output')
+    flexmock(module).should_receive('log_outputs')
 
     output = module.execute_command(full_command, input_file=input_file)
 
@@ -89,7 +137,7 @@ def test_execute_command_calls_full_command_with_shell():
     full_command = ['foo', 'bar']
     flexmock(module.os, environ={'a': 'b'})
     flexmock(module.subprocess).should_receive('Popen').with_args(
-        full_command,
+        ' '.join(full_command),
         stdin=None,
         stdout=module.subprocess.PIPE,
         stderr=module.subprocess.STDOUT,
@@ -97,7 +145,7 @@ def test_execute_command_calls_full_command_with_shell():
         env=None,
         cwd=None,
     ).and_return(flexmock(stdout=None)).once()
-    flexmock(module).should_receive('log_output')
+    flexmock(module).should_receive('log_outputs')
 
     output = module.execute_command(full_command, shell=True)
 
@@ -116,7 +164,7 @@ def test_execute_command_calls_full_command_with_extra_environment():
         env={'a': 'b', 'c': 'd'},
         cwd=None,
     ).and_return(flexmock(stdout=None)).once()
-    flexmock(module).should_receive('log_output')
+    flexmock(module).should_receive('log_outputs')
 
     output = module.execute_command(full_command, extra_environment={'c': 'd'})
 
@@ -135,13 +183,31 @@ def test_execute_command_calls_full_command_with_working_directory():
         env=None,
         cwd='/working',
     ).and_return(flexmock(stdout=None)).once()
-    flexmock(module).should_receive('log_output')
+    flexmock(module).should_receive('log_outputs')
 
     output = module.execute_command(full_command, working_directory='/working')
 
     assert output is None
 
 
+def test_execute_command_without_run_to_completion_returns_process():
+    full_command = ['foo', 'bar']
+    process = flexmock()
+    flexmock(module.os, environ={'a': 'b'})
+    flexmock(module.subprocess).should_receive('Popen').with_args(
+        full_command,
+        stdin=None,
+        stdout=module.subprocess.PIPE,
+        stderr=module.subprocess.STDOUT,
+        shell=False,
+        env=None,
+        cwd=None,
+    ).and_return(process).once()
+    flexmock(module).should_receive('log_outputs')
+
+    assert module.execute_command(full_command, run_to_completion=False) == process
+
+
 def test_execute_command_captures_output():
     full_command = ['foo', 'bar']
     expected_output = '[]'
@@ -160,7 +226,7 @@ def test_execute_command_captures_output_with_shell():
     expected_output = '[]'
     flexmock(module.os, environ={'a': 'b'})
     flexmock(module.subprocess).should_receive('check_output').with_args(
-        full_command, shell=True, env=None, cwd=None
+        'foo bar', shell=True, env=None, cwd=None
     ).and_return(flexmock(decode=lambda: expected_output)).once()
 
     output = module.execute_command(full_command, output_log_level=None, shell=True)
@@ -198,28 +264,166 @@ def test_execute_command_captures_output_with_working_directory():
     assert output == expected_output
 
 
-def test_execute_command_without_capture_does_not_raise_on_success():
-    flexmock(module.subprocess).should_receive('check_call').and_raise(
-        module.subprocess.CalledProcessError(0, 'borg init')
-    )
+def test_execute_command_with_processes_calls_full_command():
+    full_command = ['foo', 'bar']
+    processes = (flexmock(),)
+    flexmock(module.os, environ={'a': 'b'})
+    flexmock(module.subprocess).should_receive('Popen').with_args(
+        full_command,
+        stdin=None,
+        stdout=module.subprocess.PIPE,
+        stderr=module.subprocess.STDOUT,
+        shell=False,
+        env=None,
+        cwd=None,
+    ).and_return(flexmock(stdout=None)).once()
+    flexmock(module).should_receive('log_outputs')
 
-    module.execute_command_without_capture(('borg', 'init'))
+    output = module.execute_command_with_processes(full_command, processes)
+
+    assert output is None
 
 
-def test_execute_command_without_capture_does_not_raise_on_warning():
+def test_execute_command_with_processes_calls_full_command_with_output_file():
+    full_command = ['foo', 'bar']
+    processes = (flexmock(),)
+    output_file = flexmock(name='test')
+    flexmock(module.os, environ={'a': 'b'})
+    flexmock(module.subprocess).should_receive('Popen').with_args(
+        full_command,
+        stdin=None,
+        stdout=output_file,
+        stderr=module.subprocess.PIPE,
+        shell=False,
+        env=None,
+        cwd=None,
+    ).and_return(flexmock(stderr=None)).once()
+    flexmock(module).should_receive('log_outputs')
+
+    output = module.execute_command_with_processes(full_command, processes, output_file=output_file)
+
+    assert output is None
+
+
+def test_execute_command_with_processes_calls_full_command_without_capturing_output():
+    full_command = ['foo', 'bar']
+    processes = (flexmock(),)
+    flexmock(module.os, environ={'a': 'b'})
+    flexmock(module.subprocess).should_receive('Popen').with_args(
+        full_command, stdin=None, stdout=None, stderr=None, shell=False, env=None, cwd=None
+    ).and_return(flexmock(wait=lambda: 0)).once()
     flexmock(module).should_receive('exit_code_indicates_error').and_return(False)
-    flexmock(module.subprocess).should_receive('check_call').and_raise(
-        module.subprocess.CalledProcessError(1, 'borg init')
+    flexmock(module).should_receive('log_outputs')
+
+    output = module.execute_command_with_processes(
+        full_command, processes, output_file=module.DO_NOT_CAPTURE
     )
 
-    module.execute_command_without_capture(('borg', 'init'))
+    assert output is None
 
 
-def test_execute_command_without_capture_raises_on_error():
-    flexmock(module).should_receive('exit_code_indicates_error').and_return(True)
-    flexmock(module.subprocess).should_receive('check_call').and_raise(
-        module.subprocess.CalledProcessError(2, 'borg init')
+def test_execute_command_with_processes_calls_full_command_with_input_file():
+    full_command = ['foo', 'bar']
+    processes = (flexmock(),)
+    input_file = flexmock(name='test')
+    flexmock(module.os, environ={'a': 'b'})
+    flexmock(module.subprocess).should_receive('Popen').with_args(
+        full_command,
+        stdin=input_file,
+        stdout=module.subprocess.PIPE,
+        stderr=module.subprocess.STDOUT,
+        shell=False,
+        env=None,
+        cwd=None,
+    ).and_return(flexmock(stdout=None)).once()
+    flexmock(module).should_receive('log_outputs')
+
+    output = module.execute_command_with_processes(full_command, processes, input_file=input_file)
+
+    assert output is None
+
+
+def test_execute_command_with_processes_calls_full_command_with_shell():
+    full_command = ['foo', 'bar']
+    processes = (flexmock(),)
+    flexmock(module.os, environ={'a': 'b'})
+    flexmock(module.subprocess).should_receive('Popen').with_args(
+        ' '.join(full_command),
+        stdin=None,
+        stdout=module.subprocess.PIPE,
+        stderr=module.subprocess.STDOUT,
+        shell=True,
+        env=None,
+        cwd=None,
+    ).and_return(flexmock(stdout=None)).once()
+    flexmock(module).should_receive('log_outputs')
+
+    output = module.execute_command_with_processes(full_command, processes, shell=True)
+
+    assert output is None
+
+
+def test_execute_command_with_processes_calls_full_command_with_extra_environment():
+    full_command = ['foo', 'bar']
+    processes = (flexmock(),)
+    flexmock(module.os, environ={'a': 'b'})
+    flexmock(module.subprocess).should_receive('Popen').with_args(
+        full_command,
+        stdin=None,
+        stdout=module.subprocess.PIPE,
+        stderr=module.subprocess.STDOUT,
+        shell=False,
+        env={'a': 'b', 'c': 'd'},
+        cwd=None,
+    ).and_return(flexmock(stdout=None)).once()
+    flexmock(module).should_receive('log_outputs')
+
+    output = module.execute_command_with_processes(
+        full_command, processes, extra_environment={'c': 'd'}
     )
 
-    with pytest.raises(module.subprocess.CalledProcessError):
-        module.execute_command_without_capture(('borg', 'init'))
+    assert output is None
+
+
+def test_execute_command_with_processes_calls_full_command_with_working_directory():
+    full_command = ['foo', 'bar']
+    processes = (flexmock(),)
+    flexmock(module.os, environ={'a': 'b'})
+    flexmock(module.subprocess).should_receive('Popen').with_args(
+        full_command,
+        stdin=None,
+        stdout=module.subprocess.PIPE,
+        stderr=module.subprocess.STDOUT,
+        shell=False,
+        env=None,
+        cwd='/working',
+    ).and_return(flexmock(stdout=None)).once()
+    flexmock(module).should_receive('log_outputs')
+
+    output = module.execute_command_with_processes(
+        full_command, processes, working_directory='/working'
+    )
+
+    assert output is None
+
+
+def test_execute_command_with_processes_kills_processes_on_error():
+    full_command = ['foo', 'bar']
+    process = flexmock(stdout=flexmock(read=lambda count: None))
+    process.should_receive('poll')
+    process.should_receive('kill').once()
+    processes = (process,)
+    flexmock(module.os, environ={'a': 'b'})
+    flexmock(module.subprocess).should_receive('Popen').with_args(
+        full_command,
+        stdin=None,
+        stdout=module.subprocess.PIPE,
+        stderr=module.subprocess.STDOUT,
+        shell=False,
+        env=None,
+        cwd=None,
+    ).and_raise(subprocess.CalledProcessError(1, full_command, 'error')).once()
+    flexmock(module).should_receive('log_outputs').never()
+
+    with pytest.raises(subprocess.CalledProcessError):
+        module.execute_command_with_processes(full_command, processes)