Bump version for release.

NEWS

@@ -1,3 +1,35 @@
+1.5.6
+ * #292: Allow before_backup and similiar hooks to exit with a soft failure without altering the
+   monitoring status on Healthchecks or other providers. Support this by waiting to ping monitoring
+   services with a "start" status until after before_* hooks finish. Failures in before_* hooks
+   still trigger a monitoring "fail" status.
+ * #316: Fix hang when a stale database dump named pipe from an aborted borgmatic run remains on
+   disk.
+ * #323: Fix for certain configuration options like ssh_command impacting Borg invocations for
+   separate configuration files.
+ * #324: Add "borgmatic extract --strip-components" flag to remove leading path components when
+   extracting an archive.
+ * Tweak comment indentation in generated configuration file for clarity.
+ * Link to Borgmacator GNOME AppIndicator from monitoring documentation.
+
+1.5.5
+ * #314: Fix regression in support for PostgreSQL's "directory" dump format. Unlike other dump
+   formats, the "directory" dump format does not stream directly to/from Borg.
+ * #315: Fix enabled database hooks to implicitly set one_file_system configuration option to true.
+   This prevents Borg from reading devices like /dev/zero and hanging.
+ * #316: Fix hang when streaming a database dump to Borg with implicit duplicate source directories
+   by deduplicating them first.
+ * #319: Fix error message when there are no MySQL databases to dump for "all" databases.
+ * Improve documentation around the installation process. Specifically, making borgmatic commands
+   runnable via the system PATH and offering a global install option.
+
+1.5.4
+ * #310: Fix legitimate database dump command errors (exit code 1) not being treated as errors by
+   borgmatic.
+ * For database dumps, replace the named pipe on every borgmatic run. This prevent hangs on stale
+   pipes left over from previous runs.
+ * Fix error handling to handle more edge cases when executing commands.
+
 1.5.3
  * #258: Stream database dumps and restores directly to/from Borg without using any additional
    filesystem space. This feature is automatic, and works even on restores from archives made with

borgmatic/borg/check.py

@@ -134,9 +134,9 @@ def check_archives(
         # The Borg repair option trigger an interactive prompt, which won't work when output is
         # captured. And progress messes with the terminal directly.
         if repair or progress:
-            execute_command(full_command, output_file=DO_NOT_CAPTURE, error_on_warnings=True)
+            execute_command(full_command, output_file=DO_NOT_CAPTURE)
         else:
-            execute_command(full_command, error_on_warnings=True)
+            execute_command(full_command)
 
     if 'extract' in checks:
         extract.extract_last_archive_dry_run(repository, lock_wait, local_path, remote_path)

borgmatic/borg/create.py

@@ -2,6 +2,7 @@ import glob
 import itertools
 import logging
 import os
+import pathlib
 import tempfile
 
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command, execute_command_with_processes
@@ -43,6 +44,35 @@ def _expand_home_directories(directories):
     return tuple(os.path.expanduser(directory) for directory in directories)
 
 
+def deduplicate_directories(directories):
+    '''
+    Given a sequence of directories, return them as a sorted tuple with all duplicate child
+    directories removed. For instance, if paths is ('/foo', '/foo/bar'), return just: ('/foo',)
+
+    The idea is that if Borg is given a parent directory, then it doesn't also need to be given
+    child directories, because it will naturally spider the contents of the parent directory. And
+    there are cases where Borg coming across the same file twice will result in duplicate reads and
+    even hangs, e.g. when a database hook is using a named pipe for streaming database dumps to
+    Borg.
+    '''
+    deduplicated = set()
+
+    for directory in sorted(directories):
+        # If the directory is "/", that contains all child directories, so we can early out.
+        if directory == os.path.sep:
+            return (os.path.sep,)
+
+        # If no other directories are parents of current directory (even n levels up), then the
+        # current directory isn't a duplicate.
+        if not any(
+            pathlib.PurePath(other_directory) in pathlib.PurePath(directory).parents
+            for other_directory in directories
+        ):
+            deduplicated.add(directory)
+
+    return tuple(sorted(deduplicated))
+
+
 def _write_pattern_file(patterns=None):
     '''
     Given a sequence of patterns, write them to a named temporary file and return it. Return None
@@ -148,9 +178,11 @@ def create_archive(
     If a sequence of stream processes is given (instances of subprocess.Popen), then execute the
     create command while also triggering the given processes to produce output.
     '''
-    sources = _expand_directories(
-        location_config['source_directories']
-        + borgmatic_source_directories(location_config.get('borgmatic_source_directory'))
+    sources = deduplicate_directories(
+        _expand_directories(
+            location_config['source_directories']
+            + borgmatic_source_directories(location_config.get('borgmatic_source_directory'))
+        )
     )
 
     pattern_file = _write_pattern_file(location_config.get('patterns'))
@@ -175,7 +207,11 @@ def create_archive(
         + (('--chunker-params', chunker_params) if chunker_params else ())
         + (('--compression', compression) if compression else ())
         + (('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ())
-        + (('--one-file-system',) if location_config.get('one_file_system') else ())
+        + (
+            ('--one-file-system',)
+            if location_config.get('one_file_system') or stream_processes
+            else ()
+        )
         + (('--numeric-owner',) if location_config.get('numeric_owner') else ())
         + (('--noatime',) if location_config.get('atime') is False else ())
         + (('--noctime',) if location_config.get('ctime') is False else ())
@@ -215,7 +251,11 @@ def create_archive(
 
     if stream_processes:
         return execute_command_with_processes(
-            full_command, stream_processes, output_log_level, output_file, error_on_warnings=False
+            full_command,
+            stream_processes,
+            output_log_level,
+            output_file,
+            borg_local_path=local_path,
         )
 
-    return execute_command(full_command, output_log_level, output_file, error_on_warnings=False)
+    return execute_command(full_command, output_log_level, output_file, borg_local_path=local_path)

borgmatic/borg/environment.py

@@ -22,6 +22,8 @@ def initialize(storage_config):
         value = storage_config.get(option_name)
         if value:
             os.environ[environment_variable_name] = value
+        else:
+            os.environ.pop(environment_variable_name, None)
 
     for (
         option_name,

borgmatic/borg/extract.py

@@ -28,7 +28,9 @@ def extract_last_archive_dry_run(repository, lock_wait=None, local_path='borg',
         + (repository,)
     )
 
-    list_output = execute_command(full_list_command, output_log_level=None, error_on_warnings=False)
+    list_output = execute_command(
+        full_list_command, output_log_level=None, borg_local_path=local_path
+    )
 
     try:
         last_archive_name = list_output.strip().splitlines()[-1]
@@ -49,7 +51,7 @@ def extract_last_archive_dry_run(repository, lock_wait=None, local_path='borg',
         )
     )
 
-    execute_command(full_extract_command, working_directory=None, error_on_warnings=True)
+    execute_command(full_extract_command, working_directory=None)
 
 
 def extract_archive(
@@ -62,8 +64,8 @@ def extract_archive(
     local_path='borg',
     remote_path=None,
     destination_path=None,
+    strip_components=None,
     progress=False,
-    error_on_warnings=True,
     extract_to_stdout=False,
 ):
     '''
@@ -90,6 +92,7 @@ def extract_archive(
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--dry-run',) if dry_run else ())
+        + (('--strip-components', str(strip_components)) if strip_components else ())
         + (('--progress',) if progress else ())
         + (('--stdout',) if extract_to_stdout else ())
         + ('::'.join((repository if ':' in repository else os.path.abspath(repository), archive)),)
@@ -100,10 +103,7 @@ def extract_archive(
     # the terminal directly.
     if progress:
         return execute_command(
-            full_command,
-            output_file=DO_NOT_CAPTURE,
-            working_directory=destination_path,
-            error_on_warnings=error_on_warnings,
+            full_command, output_file=DO_NOT_CAPTURE, working_directory=destination_path
         )
         return None
 
@@ -112,12 +112,9 @@ def extract_archive(
             full_command,
             output_file=subprocess.PIPE,
             working_directory=destination_path,
-            error_on_warnings=error_on_warnings,
             run_to_completion=False,
         )
 
-    # Error on warnings by default, as Borg only gives a warning if the restore paths don't exist in
-    # the archive!
-    execute_command(
-        full_command, working_directory=destination_path, error_on_warnings=error_on_warnings
-    )
+    # Don't give Borg local path, so as to error on warnings, as Borg only gives a warning if the
+    # restore paths don't exist in the archive!
+    execute_command(full_command, working_directory=destination_path)

borgmatic/borg/info.py

@@ -41,5 +41,5 @@ def display_archives_info(
     return execute_command(
         full_command,
         output_log_level=None if info_arguments.json else logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path=local_path,
     )

borgmatic/borg/init.py

@@ -55,4 +55,4 @@ def initialize_repository(
     )
 
     # Do not capture output here, so as to support interactive prompts.
-    execute_command(init_command, output_file=DO_NOT_CAPTURE, error_on_warnings=False)
+    execute_command(init_command, output_file=DO_NOT_CAPTURE, borg_local_path=local_path)

borgmatic/borg/list.py

@@ -36,7 +36,7 @@ def resolve_archive_name(repository, archive, storage_config, local_path='borg',
         + ('--short', repository)
     )
 
-    output = execute_command(full_command, output_log_level=None, error_on_warnings=False)
+    output = execute_command(full_command, output_log_level=None, borg_local_path=local_path)
     try:
         latest_archive = output.strip().splitlines()[-1]
     except IndexError:
@@ -85,5 +85,5 @@ def list_archives(repository, storage_config, list_arguments, local_path='borg',
     return execute_command(
         full_command,
         output_log_level=None if list_arguments.json else logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path=local_path,
     )

borgmatic/borg/mount.py

@@ -40,7 +40,7 @@ def mount_archive(
 
     # Don't capture the output when foreground mode is used so that ctrl-C can work properly.
     if foreground:
-        execute_command(full_command, output_file=DO_NOT_CAPTURE, error_on_warnings=False)
+        execute_command(full_command, output_file=DO_NOT_CAPTURE, borg_local_path=local_path)
         return
 
-    execute_command(full_command, error_on_warnings=False)
+    execute_command(full_command, borg_local_path=local_path)

borgmatic/borg/prune.py

@@ -72,4 +72,4 @@ def prune_archives(
     else:
         output_log_level = logging.INFO
 
-    execute_command(full_command, output_log_level=output_log_level, error_on_warnings=False)
+    execute_command(full_command, output_log_level=output_log_level, borg_local_path=local_path)

borgmatic/borg/umount.py

@@ -17,4 +17,4 @@ def unmount_archive(mount_point, local_path='borg'):
         + (mount_point,)
     )
 
-    execute_command(full_command, error_on_warnings=True)
+    execute_command(full_command)

borgmatic/commands/arguments.py

@@ -340,6 +340,13 @@ def parse_arguments(*unparsed_arguments):
         dest='destination',
         help='Directory to extract files into, defaults to the current directory',
     )
+    extract_group.add_argument(
+        '--strip-components',
+        type=int,
+        metavar='NUMBER',
+        dest='strip_components',
+        help='Number of leading path components to remove from each extracted path. Skip paths with fewer elements',
+    )
     extract_group.add_argument(
         '--progress',
         dest='progress',

borgmatic/commands/borgmatic.py

@@ -59,11 +59,10 @@ def run_configuration(config_filename, config, arguments):
     try:
         if prune_create_or_check:
             dispatch.call_hooks(
-                'ping_monitor',
+                'initialize_monitor',
                 hooks,
                 config_filename,
                 monitor.MONITOR_HOOK_NAMES,
-                monitor.State.START,
                 monitoring_log_level,
                 global_arguments.dry_run,
             )
@@ -91,6 +90,16 @@ def run_configuration(config_filename, config, arguments):
                 'pre-check',
                 global_arguments.dry_run,
             )
+        if prune_create_or_check:
+            dispatch.call_hooks(
+                'ping_monitor',
+                hooks,
+                config_filename,
+                monitor.MONITOR_HOOK_NAMES,
+                monitor.State.START,
+                monitoring_log_level,
+                global_arguments.dry_run,
+            )
     except (OSError, CalledProcessError) as error:
         if command.considered_soft_failure(config_filename, error):
             return
@@ -123,6 +132,16 @@ def run_configuration(config_filename, config, arguments):
 
     if not encountered_error:
         try:
+            if prune_create_or_check:
+                dispatch.call_hooks(
+                    'ping_monitor',
+                    hooks,
+                    config_filename,
+                    monitor.MONITOR_HOOK_NAMES,
+                    monitor.State.FINISH,
+                    monitoring_log_level,
+                    global_arguments.dry_run,
+                )
             if 'prune' in arguments:
                 command.execute_hook(
                     hooks.get('after_prune'),
@@ -155,16 +174,6 @@ def run_configuration(config_filename, config, arguments):
                     'post-check',
                     global_arguments.dry_run,
                 )
-            if {'prune', 'create', 'check'}.intersection(arguments):
-                dispatch.call_hooks(
-                    'ping_monitor',
-                    hooks,
-                    config_filename,
-                    monitor.MONITOR_HOOK_NAMES,
-                    monitor.State.FINISH,
-                    monitoring_log_level,
-                    global_arguments.dry_run,
-                )
         except (OSError, CalledProcessError) as error:
             if command.considered_soft_failure(config_filename, error):
                 return
@@ -176,6 +185,15 @@ def run_configuration(config_filename, config, arguments):
 
     if encountered_error and prune_create_or_check:
         try:
+            dispatch.call_hooks(
+                'ping_monitor',
+                hooks,
+                config_filename,
+                monitor.MONITOR_HOOK_NAMES,
+                monitor.State.FAIL,
+                monitoring_log_level,
+                global_arguments.dry_run,
+            )
             command.execute_hook(
                 hooks.get('on_error'),
                 hooks.get('umask'),
@@ -186,15 +204,6 @@ def run_configuration(config_filename, config, arguments):
                 error=encountered_error,
                 output=getattr(encountered_error, 'output', ''),
             )
-            dispatch.call_hooks(
-                'ping_monitor',
-                hooks,
-                config_filename,
-                monitor.MONITOR_HOOK_NAMES,
-                monitor.State.FAIL,
-                monitoring_log_level,
-                global_arguments.dry_run,
-            )
         except (OSError, CalledProcessError) as error:
             if command.considered_soft_failure(config_filename, error):
                 return
@@ -254,6 +263,14 @@ def run_actions(
         )
     if 'create' in arguments:
         logger.info('{}: Creating archive{}'.format(repository, dry_run_label))
+        dispatch.call_hooks(
+            'remove_database_dumps',
+            hooks,
+            repository,
+            dump.DATABASE_HOOK_NAMES,
+            location,
+            global_arguments.dry_run,
+        )
         active_dumps = dispatch.call_hooks(
             'dump_databases',
             hooks,
@@ -311,6 +328,7 @@ def run_actions(
                 local_path=local_path,
                 remote_path=remote_path,
                 destination_path=arguments['extract'].destination,
+                strip_components=arguments['extract'].strip_components,
                 progress=arguments['extract'].progress,
             )
     if 'mount' in arguments:
@@ -346,6 +364,14 @@ def run_actions(
                     repository, arguments['restore'].archive
                 )
             )
+            dispatch.call_hooks(
+                'remove_database_dumps',
+                hooks,
+                repository,
+                dump.DATABASE_HOOK_NAMES,
+                location,
+                global_arguments.dry_run,
+            )
 
             restore_names = arguments['restore'].databases or []
             if 'all' in restore_names:
@@ -386,10 +412,12 @@ def run_actions(
                         local_path=local_path,
                         remote_path=remote_path,
                         destination_path='/',
-                        extract_to_stdout=True,
+                        # A directory format dump isn't a single file, and therefore can't extract
+                        # to stdout. In this case, the extract_process return value is None.
+                        extract_to_stdout=bool(restore_database.get('format') != 'directory'),
                     )
 
-                    # Run a single database restore, consuming the extract stdout.
+                    # Run a single database restore, consuming the extract stdout (if any).
                     dispatch.call_hooks(
                         'restore_database_dump',
                         {hook_name: [restore_database]},
@@ -400,6 +428,15 @@ def run_actions(
                         extract_process,
                     )
 
+            dispatch.call_hooks(
+                'remove_database_dumps',
+                hooks,
+                repository,
+                dump.DATABASE_HOOK_NAMES,
+                location,
+                global_arguments.dry_run,
+            )
+
             if not restore_names and not found_names:
                 raise ValueError('No databases were found to restore')
 

borgmatic/config/generate.py

@@ -37,9 +37,7 @@ def _schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
                 for item_schema in schema['seq']
             ]
         )
-        add_comments_to_configuration_sequence(
-            config, schema, indent=(level * INDENT) + SEQUENCE_INDENT
-        )
+        add_comments_to_configuration_sequence(config, schema, indent=(level * INDENT))
     elif 'map' in schema:
         config = yaml.comments.CommentedMap(
             [
@@ -86,8 +84,8 @@ def _comment_out_optional_configuration(rendered_config):
     optional = False
 
     for line in rendered_config.split('\n'):
-        # Upon encountering an optional configuration option, commenting out lines until the next
-        # blank line.
+        # Upon encountering an optional configuration option, comment out lines until the next blank
+        # line.
         if line.strip().startswith('# {}'.format(COMMENTED_OUT_SENTINEL)):
             optional = True
             continue
@@ -142,7 +140,7 @@ def add_comments_to_configuration_sequence(config, schema, indent=0):
 
     ```
     things:
-          # First key description. Added by this function.
+        # First key description. Added by this function.
         - key: foo
           # Second key description. Added by add_comments_to_configuration_map().
           other: bar

borgmatic/config/schema.yaml

@@ -30,7 +30,10 @@ map:
                     - user@backupserver:sourcehostname.borg
             one_file_system:
                 type: bool
-                desc: Stay in same file system (do not cross mount points). Defaults to false.
+                desc: |
+                  Stay in same file system (do not cross mount points). Defaults to false. But when
+                  a database hook is used, the setting here is ignored and one_file_system is
+                  considered true.
                 example: true
             numeric_owner:
                 type: bool
@@ -53,7 +56,9 @@ map:
                 desc: |
                     Use Borg's --read-special flag to allow backup of block and other special
                     devices. Use with caution, as it will lead to problems if used when
-                    backing up special devices such as /dev/zero. Defaults to false.
+                    backing up special devices such as /dev/zero. Defaults to false. But when a
+                    database hook is used, the setting here is ignored and read_special is
+                    considered true.
                 example: false
             bsd_flags:
                 type: bool
@@ -452,7 +457,9 @@ map:
                             type: str
                             desc: |
                                 Database name (required if using this hook). Or "all" to dump all
-                                databases on the host.
+                                databases on the host. Note that using this database hook implicitly
+                                enables both read_special and one_file_system (see above) to support
+                                dump and restore streaming.
                             example: users
                         hostname:
                             type: str
@@ -508,7 +515,9 @@ map:
                             type: str
                             desc: |
                                 Database name (required if using this hook). Or "all" to dump all
-                                databases on the host.
+                                databases on the host. Note that using this database hook implicitly
+                                enables both read_special and one_file_system (see above) to support
+                                dump and restore streaming.
                             example: users
                         hostname:
                             type: str

borgmatic/execute.py

@@ -11,15 +11,21 @@ ERROR_OUTPUT_MAX_LINE_COUNT = 25
 BORG_ERROR_EXIT_CODE = 2
 
 
-def exit_code_indicates_error(exit_code, error_on_warnings=True):
+def exit_code_indicates_error(process, exit_code, borg_local_path=None):
     '''
-    Return True if the given exit code from running a command corresponds to an error. If error on
-    warnings is False, then treat exit code 1 as a warning instead of an error.
+    Return True if the given exit code from running a command corresponds to an error. If a Borg
+    local path is given and matches the process' command, then treat exit code 1 as a warning
+    instead of an error.
     '''
-    if error_on_warnings:
-        return bool(exit_code != 0)
+    if exit_code is None:
+        return False
 
-    return bool(exit_code >= BORG_ERROR_EXIT_CODE)
+    command = process.args.split(' ') if isinstance(process.args, str) else process.args
+
+    if borg_local_path and command[0] == borg_local_path:
+        return bool(exit_code >= BORG_ERROR_EXIT_CODE)
+
+    return bool(exit_code != 0)
 
 
 def command_for_process(process):
@@ -39,11 +45,11 @@ def output_buffer_for_process(process, exclude_stdouts):
     return process.stderr if process.stdout in exclude_stdouts else process.stdout
 
 
-def log_outputs(processes, exclude_stdouts, output_log_level, error_on_warnings):
+def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
     '''
     Given a sequence of subprocess.Popen() instances for multiple processes, log the output for each
     process with the requested log level. Additionally, raise a CalledProcessError if a process
-    exits with an error (or a warning, if error on warnings is True).
+    exits with an error (or a warning for exit code 1, if that process matches the Borg local path).
 
     For simplicity, it's assumed that the output buffer for each process is its stdout. But if any
     stdouts are given to exclude, then for any matching processes, log from their stderr instead.
@@ -61,24 +67,46 @@ def log_outputs(processes, exclude_stdouts, output_log_level, error_on_warnings)
 
     # Log output for each process until they all exit.
     while True:
-        if not output_buffers:
-            break
+        if output_buffers:
+            (ready_buffers, _, _) = select.select(output_buffers, [], [])
 
-        (ready_buffers, _, _) = select.select(output_buffers, [], [])
+            for ready_buffer in ready_buffers:
+                line = ready_buffer.readline().rstrip().decode()
+                if not line:
+                    continue
 
-        for ready_buffer in ready_buffers:
-            line = ready_buffer.readline().rstrip().decode()
-            if not line:
-                continue
+                # Keep the last few lines of output in case the process errors, and we need the output for
+                # the exception below.
+                last_lines = buffer_last_lines[ready_buffer]
+                last_lines.append(line)
+                if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT:
+                    last_lines.pop(0)
 
-            # Keep the last few lines of output in case the process errors, and we need the output for
-            # the exception below.
-            last_lines = buffer_last_lines[ready_buffer]
-            last_lines.append(line)
-            if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT:
-                last_lines.pop(0)
+                logger.log(output_log_level, line)
 
-            logger.log(output_log_level, line)
+        for process in processes:
+            exit_code = process.poll() if output_buffers else process.wait()
+
+            # If any process errors, then raise accordingly.
+            if exit_code_indicates_error(process, exit_code, borg_local_path):
+                # If an error occurs, include its output in the raised exception so that we don't
+                # inadvertently hide error output.
+                output_buffer = output_buffer_for_process(process, exclude_stdouts)
+
+                last_lines = buffer_last_lines[output_buffer] if output_buffer else []
+                if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT:
+                    last_lines.insert(0, '...')
+
+                # Something has gone wrong. So vent each process' output buffer to prevent it from
+                # hanging. And then kill the process.
+                for other_process in processes:
+                    if other_process.poll() is None:
+                        other_process.stdout.read(0)
+                        other_process.kill()
+
+                raise subprocess.CalledProcessError(
+                    exit_code, command_for_process(process), '\n'.join(last_lines)
+                )
 
         if all(process.poll() is not None for process in processes):
             break
@@ -95,23 +123,6 @@ def log_outputs(processes, exclude_stdouts, output_log_level, error_on_warnings)
         if remaining_output:  # pragma: no cover
             logger.log(output_log_level, remaining_output)
 
-    # If any process errored, then raise accordingly.
-    for process in processes:
-        exit_code = process.wait()
-
-        if exit_code_indicates_error(exit_code, error_on_warnings):
-            # If an error occurs, include its output in the raised exception so that we don't
-            # inadvertently hide error output.
-            output_buffer = output_buffer_for_process(process, exclude_stdouts)
-
-            last_lines = buffer_last_lines[output_buffer] if output_buffer else []
-            if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT:
-                last_lines.insert(0, '...')
-
-            raise subprocess.CalledProcessError(
-                exit_code, command_for_process(process), '\n'.join(last_lines)
-            )
-
 
 def log_command(full_command, input_file, output_file):
     '''
@@ -139,7 +150,7 @@ def execute_command(
     shell=False,
     extra_environment=None,
     working_directory=None,
-    error_on_warnings=True,
+    borg_local_path=None,
     run_to_completion=True,
 ):
     '''
@@ -150,9 +161,9 @@ def execute_command(
     then read stdin from the file. If shell is True, execute the command within a shell. If an extra
     environment dict is given, then use it to augment the current environment, and pass the result
     into the command. If a working directory is given, use that as the present working directory
-    when running the command. If error on warnings is False, then treat exit code 1 as a warning
-    instead of an error. If run to completion is False, then return the process for the command
-    without executing it to completion.
+    when running the command. If a Borg local path is given, and the command matches it (regardless
+    of arguments), treat exit code 1 as a warning instead of an error. If run to completion is
+    False, then return the process for the command without executing it to completion.
 
     Raise subprocesses.CalledProcessError if an error occurs while running the command.
     '''
@@ -179,7 +190,9 @@ def execute_command(
     if not run_to_completion:
         return process
 
-    log_outputs((process,), (input_file, output_file), output_log_level, error_on_warnings)
+    log_outputs(
+        (process,), (input_file, output_file), output_log_level, borg_local_path=borg_local_path
+    )
 
 
 def execute_command_with_processes(
@@ -191,7 +204,7 @@ def execute_command_with_processes(
     shell=False,
     extra_environment=None,
     working_directory=None,
-    error_on_warnings=True,
+    borg_local_path=None,
 ):
     '''
     Execute the given command (a sequence of command/argument strings) and log its output at the
@@ -204,8 +217,8 @@ def execute_command_with_processes(
     the file.  If shell is True, execute the command within a shell. If an extra environment dict is
     given, then use it to augment the current environment, and pass the result into the command. If
     a working directory is given, use that as the present working directory when running the
-    command.  If error on warnings is False, then treat exit code 1 as a warning instead of an
-    error.
+    command. If a Borg local path is given, then for any matching command or process (regardless of
+    arguments), treat exit code 1 as a warning instead of an error.
 
     Raise subprocesses.CalledProcessError if an error occurs while running the command or in the
     upstream process.
@@ -240,5 +253,5 @@ def execute_command_with_processes(
         tuple(processes) + (command_process,),
         (input_file, output_file),
         output_log_level,
-        error_on_warnings,
+        borg_local_path=borg_local_path,
     )

borgmatic/hooks/cronhub.py

@@ -13,6 +13,15 @@ MONITOR_STATE_TO_CRONHUB = {
 }
 
 
+def initialize_monitor(
+    ping_url, config_filename, monitoring_log_level, dry_run
+):  # pragma: no cover
+    '''
+    No initialization is necessary for this monitor.
+    '''
+    pass
+
+
 def ping_monitor(ping_url, config_filename, state, monitoring_log_level, dry_run):
     '''
     Ping the given Cronhub URL, modified with the monitor.State. Use the given configuration

borgmatic/hooks/cronitor.py

@@ -13,6 +13,15 @@ MONITOR_STATE_TO_CRONITOR = {
 }
 
 
+def initialize_monitor(
+    ping_url, config_filename, monitoring_log_level, dry_run
+):  # pragma: no cover
+    '''
+    No initialization is necessary for this monitor.
+    '''
+    pass
+
+
 def ping_monitor(ping_url, config_filename, state, monitoring_log_level, dry_run):
     '''
     Ping the given Cronitor URL, modified with the monitor.State. Use the given configuration

borgmatic/hooks/dump.py

@@ -33,53 +33,39 @@ def make_database_dump_filename(dump_path, name, hostname=None):
     return os.path.join(os.path.expanduser(dump_path), hostname or 'localhost', name)
 
 
+def create_parent_directory_for_dump(dump_path):
+    '''
+    Create a directory to contain the given dump path.
+    '''
+    os.makedirs(os.path.dirname(dump_path), mode=0o700, exist_ok=True)
+
+
 def create_named_pipe_for_dump(dump_path):
     '''
     Create a named pipe at the given dump path.
     '''
-    os.makedirs(os.path.dirname(dump_path), mode=0o700, exist_ok=True)
-    if not os.path.exists(dump_path):
-        os.mkfifo(dump_path, mode=0o600)
+    create_parent_directory_for_dump(dump_path)
+    os.mkfifo(dump_path, mode=0o600)
 
 
-def remove_database_dumps(dump_path, databases, database_type_name, log_prefix, dry_run):
+def remove_database_dumps(dump_path, database_type_name, log_prefix, dry_run):
     '''
-    Remove the database dumps for the given databases in the dump directory path. The databases are
-    supplied as a sequence of dicts, one dict describing each database as per the configuration
-    schema. Use the name of the database type and the log prefix in any log entries. If this is a
-    dry run, then don't actually remove anything.
+    Remove all database dumps in the given dump directory path (including the directory itself). If
+    this is a dry run, then don't actually remove anything.
     '''
-    if not databases:
-        logger.debug('{}: No {} databases configured'.format(log_prefix, database_type_name))
-        return
-
     dry_run_label = ' (dry run; not actually removing anything)' if dry_run else ''
 
     logger.info(
         '{}: Removing {} database dumps{}'.format(log_prefix, database_type_name, dry_run_label)
     )
 
-    for database in databases:
-        dump_filename = make_database_dump_filename(
-            dump_path, database['name'], database.get('hostname')
-        )
+    expanded_path = os.path.expanduser(dump_path)
 
-        logger.debug(
-            '{}: Removing {} database dump {} from {}{}'.format(
-                log_prefix, database_type_name, database['name'], dump_filename, dry_run_label
-            )
-        )
-        if dry_run:
-            continue
+    if dry_run:
+        return
 
-        if os.path.isdir(dump_filename):
-            shutil.rmtree(dump_filename)
-        else:
-            os.remove(dump_filename)
-        dump_file_dir = os.path.dirname(dump_filename)
-
-        if len(os.listdir(dump_file_dir)) == 0:
-            os.rmdir(dump_file_dir)
+    if os.path.exists(expanded_path):
+        shutil.rmtree(expanded_path)
 
 
 def convert_glob_patterns_to_borg_patterns(patterns):

borgmatic/hooks/healthchecks.py

@@ -65,20 +65,24 @@ def format_buffered_logs_for_payload():
     return payload
 
 
+def initialize_monitor(
+    ping_url_or_uuid, config_filename, monitoring_log_level, dry_run
+):  # pragma: no cover
+    '''
+    Add a handler to the root logger that stores in memory the most recent logs emitted. That
+    way, we can send them all to Healthchecks upon a finish or failure state.
+    '''
+    logging.getLogger().addHandler(
+        Forgetful_buffering_handler(PAYLOAD_LIMIT_BYTES, monitoring_log_level)
+    )
+
+
 def ping_monitor(ping_url_or_uuid, config_filename, state, monitoring_log_level, dry_run):
     '''
     Ping the given Healthchecks URL or UUID, modified with the monitor.State. Use the given
     configuration filename in any log entries, and log to Healthchecks with the giving log level.
     If this is a dry run, then don't actually ping anything.
     '''
-    if state is monitor.State.START:
-        # Add a handler to the root logger that stores in memory the most recent logs emitted. That
-        # way, we can send them all to Healthchecks upon a finish or failure state.
-        logging.getLogger().addHandler(
-            Forgetful_buffering_handler(PAYLOAD_LIMIT_BYTES, monitoring_log_level)
-        )
-        payload = ''
-
     ping_url = (
         ping_url_or_uuid
         if ping_url_or_uuid.startswith('http')
@@ -97,6 +101,8 @@ def ping_monitor(ping_url_or_uuid, config_filename, state, monitoring_log_level,
 
     if state in (monitor.State.FINISH, monitor.State.FAIL):
         payload = format_buffered_logs_for_payload()
+    else:
+        payload = ''
 
     if not dry_run:
         logging.getLogger('urllib3').setLevel(logging.ERROR)

borgmatic/hooks/mysql.py

@@ -73,9 +73,11 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
             make_dump_path(location_config), requested_name, database.get('hostname')
         )
         extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
-        dump_command_names = database_names_to_dump(
+        dump_database_names = database_names_to_dump(
             database, extra_environment, log_prefix, dry_run_label
         )
+        if not dump_database_names:
+            raise ValueError('Cannot find any MySQL databases to dump.')
 
         dump_command = (
             ('mysqldump',)
@@ -86,7 +88,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
             + (('--user', database['username']) if 'username' in database else ())
             + (tuple(database['options'].split(' ')) if 'options' in database else ())
             + ('--databases',)
-            + dump_command_names
+            + dump_database_names
             # Use shell redirection rather than execute_command(output_file=open(...)) to prevent
             # the open() call on a named pipe from hanging the main borgmatic process.
             + ('>', dump_filename)
@@ -116,14 +118,11 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
 
 def remove_database_dumps(databases, log_prefix, location_config, dry_run):  # pragma: no cover
     '''
-    Remove the database dumps for the given databases. The databases are supplied as a sequence of
-    dicts, one dict describing each database as per the configuration schema. Use the log prefix in
-    any log entries. Use the given location configuration dict to construct the destination path. If
-    this is a dry run, then don't actually remove anything.
+    Remove all database dump files for this hook regardless of the given databases. Use the log
+    prefix in any log entries. Use the given location configuration dict to construct the
+    destination path. If this is a dry run, then don't actually remove anything.
     '''
-    dump.remove_database_dumps(
-        make_dump_path(location_config), databases, 'MySQL', log_prefix, dry_run
-    )
+    dump.remove_database_dumps(make_dump_path(location_config), 'MySQL', log_prefix, dry_run)
 
 
 def make_database_dump_pattern(
@@ -172,4 +171,5 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
         extra_environment=extra_environment,
+        borg_local_path=location_config.get('local_path', 'borg'),
     )

borgmatic/hooks/pagerduty.py

@@ -12,6 +12,15 @@ logger = logging.getLogger(__name__)
 EVENTS_API_URL = 'https://events.pagerduty.com/v2/enqueue'
 
 
+def initialize_monitor(
+    integration_key, config_filename, monitoring_log_level, dry_run
+):  # pragma: no cover
+    '''
+    No initialization is necessary for this monitor.
+    '''
+    pass
+
+
 def ping_monitor(integration_key, config_filename, state, monitoring_log_level, dry_run):
     '''
     If this is an error state, create a PagerDuty event with the given integration key. Use the

borgmatic/hooks/postgresql.py

@@ -36,6 +36,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
             make_dump_path(location_config), name, database.get('hostname')
         )
         all_databases = bool(name == 'all')
+        dump_format = database.get('format', 'custom')
         command = (
             (
                 'pg_dumpall' if all_databases else 'pg_dump',
@@ -46,12 +47,14 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
             + (('--host', database['hostname']) if 'hostname' in database else ())
             + (('--port', str(database['port'])) if 'port' in database else ())
             + (('--username', database['username']) if 'username' in database else ())
-            + (() if all_databases else ('--format', database.get('format', 'custom')))
+            + (() if all_databases else ('--format', dump_format))
+            + (('--file', dump_filename) if dump_format == 'directory' else ())
             + (tuple(database['options'].split(' ')) if 'options' in database else ())
             + (() if all_databases else (name,))
             # Use shell redirection rather than the --file flag to sidestep synchronization issues
-            # when pg_dump/pg_dumpall tries to write to a named pipe.
-            + ('>', dump_filename)
+            # when pg_dump/pg_dumpall tries to write to a named pipe. But for the directory dump
+            # format in a particular, a named destination is required, and redirection doesn't work.
+            + (('>', dump_filename) if dump_format != 'directory' else ())
         )
         extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
 
@@ -63,7 +66,10 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
         if dry_run:
             continue
 
-        dump.create_named_pipe_for_dump(dump_filename)
+        if dump_format == 'directory':
+            dump.create_parent_directory_for_dump(dump_filename)
+        else:
+            dump.create_named_pipe_for_dump(dump_filename)
 
         processes.append(
             execute_command(
@@ -76,14 +82,11 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
 
 def remove_database_dumps(databases, log_prefix, location_config, dry_run):  # pragma: no cover
     '''
-    Remove the database dumps for the given databases. The databases are supplied as a sequence of
-    dicts, one dict describing each database as per the configuration schema. Use the log prefix in
-    any log entries. Use the given location configuration dict to construct the destination path. If
-    this is a dry run, then don't actually remove anything.
+    Remove all database dump files for this hook regardless of the given databases. Use the log
+    prefix in any log entries. Use the given location configuration dict to construct the
+    destination path. If this is a dry run, then don't actually remove anything.
     '''
-    dump.remove_database_dumps(
-        make_dump_path(location_config), databases, 'PostgreSQL', log_prefix, dry_run
-    )
+    dump.remove_database_dumps(make_dump_path(location_config), 'PostgreSQL', log_prefix, dry_run)
 
 
 def make_database_dump_pattern(
@@ -104,6 +107,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
     Use the given log prefix in any log entries. If this is a dry run, then don't actually restore
     anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce
     output to consume.
+
+    If the extract process is None, then restore the dump from the filesystem rather than from an
+    extract stream.
     '''
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
 
@@ -112,6 +118,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
 
     database = database_config[0]
     all_databases = bool(database['name'] == 'all')
+    dump_filename = dump.make_database_dump_filename(
+        make_dump_path(location_config), database['name'], database.get('hostname')
+    )
     analyze_command = (
         ('psql', '--no-password', '--quiet')
         + (('--host', database['hostname']) if 'hostname' in database else ())
@@ -130,6 +139,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--username', database['username']) if 'username' in database else ())
+        + (() if extract_process else (dump_filename,))
     )
     extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
 
@@ -141,9 +151,10 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
 
     execute_command_with_processes(
         restore_command,
-        [extract_process],
+        [extract_process] if extract_process else [],
         output_log_level=logging.DEBUG,
-        input_file=extract_process.stdout,
+        input_file=extract_process.stdout if extract_process else None,
         extra_environment=extra_environment,
+        borg_local_path=location_config.get('local_path', 'borg'),
     )
     execute_command(analyze_command, extra_environment=extra_environment)

docs/how-to/backup-your-databases.md

@@ -24,12 +24,17 @@ hooks:
 
 As part of each backup, borgmatic streams a database dump for each configured
 database directly to Borg, so it's included in the backup without consuming
-additional disk space.
+additional disk space. (The one exception is PostgreSQL's "directory" dump
+format, which can't stream and therefore does consume temporary disk space.)
 
 To support this, borgmatic creates temporary named pipes in `~/.borgmatic` by
 default. To customize this path, set the `borgmatic_source_directory` option
 in the `location` section of borgmatic's configuration.
 
+Also note that using a database hook implicitly enables both the
+`read_special` and `one_file_system` configuration settings (even if they're
+disabled in your configuration) to support this dump and restore streaming.
+
 Here's a more involved example that connects to remote databases:
 
 ```yaml

docs/how-to/monitor-your-backups.md

@@ -42,6 +42,7 @@ below for how to configure this.
 software to consume borgmatic JSON output and track when the last
 successful backup occurred. See [scripting
 borgmatic](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#scripting-borgmatic)
+and [related software](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#related-software)
 below for how to configure this.
 5. **Borg hosting providers**: Most [Borg hosting
 providers](https://torsion.org/borgmatic/#hosting-providers) include
@@ -116,21 +117,21 @@ hooks:
 ```
 
 With this hook in place, borgmatic pings your Healthchecks project when a
-backup begins, ends, or errors. Specifically, before the <a
+backup begins, ends, or errors. Specifically, after the <a
 href="https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/">`before_backup`
 hooks</a> run, borgmatic lets Healthchecks know that it has started if any of
 the `prune`, `create`, or `check` actions are run.
 
 Then, if the actions complete successfully, borgmatic notifies Healthchecks of
-the success after the `after_backup` hooks run, and includes borgmatic logs in
+the success before the `after_backup` hooks run, and includes borgmatic logs in
 the payload data sent to Healthchecks. This means that borgmatic logs show up
 in the Healthchecks UI, although be aware that Healthchecks currently has a
 10-kilobyte limit for the logs in each ping.
 
-If an error occurs during any action, borgmatic notifies Healthchecks after
-the `on_error` hooks run, also tacking on logs including the error itself. But
-the logs are only included for errors that occur when a `prune`, `create`, or
-`check` action is run.
+If an error occurs during any action or hook, borgmatic notifies Healthchecks
+before the `on_error` hooks run, also tacking on logs including the error
+itself. But the logs are only included for errors that occur when a `prune`,
+`create`, or `check` action is run.
 
 You can customize the verbosity of the logs that are sent to Healthchecks with
 borgmatic's `--monitoring-verbosity` flag. The `--files` and `--stats` flags
@@ -156,13 +157,13 @@ hooks:
 ```
 
 With this hook in place, borgmatic pings your Cronitor monitor when a backup
-begins, ends, or errors. Specifically, before the <a
+begins, ends, or errors. Specifically, after the <a
 href="https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/">`before_backup`
 hooks</a> run, borgmatic lets Cronitor know that it has started if any of the
 `prune`, `create`, or `check` actions are run. Then, if the actions complete
-successfully, borgmatic notifies Cronitor of the success after the
-`after_backup` hooks run. And if an error occurs during any action, borgmatic
-notifies Cronitor after the `on_error` hooks run.
+successfully, borgmatic notifies Cronitor of the success before the
+`after_backup` hooks run. And if an error occurs during any action or hook,
+borgmatic notifies Cronitor before the `on_error` hooks run.
 
 You can configure Cronitor to notify you by a [variety of
 mechanisms](https://cronitor.io/docs/cron-job-notifications) when backups fail
@@ -184,13 +185,13 @@ hooks:
 ```
 
 With this hook in place, borgmatic pings your Cronhub monitor when a backup
-begins, ends, or errors. Specifically, before the <a
+begins, ends, or errors. Specifically, after the <a
 href="https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/">`before_backup`
 hooks</a> run, borgmatic lets Cronhub know that it has started if any of the
 `prune`, `create`, or `check` actions are run. Then, if the actions complete
-successfully, borgmatic notifies Cronhub of the success after the
-`after_backup` hooks run. And if an error occurs during any action, borgmatic
-notifies Cronhub after the `on_error` hooks run.
+successfully, borgmatic notifies Cronhub of the success before the
+`after_backup` hooks run. And if an error occurs during any action or hook,
+borgmatic notifies Cronhub before the `on_error` hooks run.
 
 Note that even though you configure borgmatic with the "start" variant of the
 ping URL, borgmatic substitutes the correct state into the URL when pinging
@@ -227,7 +228,7 @@ hooks:
 
 With this hook in place, borgmatic creates a PagerDuty event for your service
 whenever backups fail. Specifically, if an error occurs during a `create`,
-`prune`, or `check` action, borgmatic sends an event to PagerDuty after the
+`prune`, or `check` action, borgmatic sends an event to PagerDuty before the
 `on_error` hooks run. Note that borgmatic does not contact PagerDuty when a
 backup starts or ends without error.
 
@@ -250,6 +251,11 @@ suppressed so as not to interfere with the captured JSON. Also note that JSON
 output only shows up at the console, and not in syslog.
 
 
+## Related software
+
+ * [Borgmacator GNOME AppIndicator](https://github.com/N-Coder/borgmacator/)
+
+
 ### Successful backups
 
 `borgmatic list` includes support for a `--successful` flag that only lists

docs/how-to/set-up-backups.md

@@ -12,30 +12,66 @@ First, [install
 Borg](https://borgbackup.readthedocs.io/en/stable/installation.html), at least
 version 1.1.
 
-Then, download and install borgmatic by running the following command:
+Then, download and install borgmatic as a [user site
+installation](https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site)
+by running the following command:
 
 ```bash
 sudo pip3 install --user --upgrade borgmatic
 ```
 
-This is a [recommended user site
-installation](https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site).
-You will need to ensure that `/root/.local/bin` is available on your `$PATH`
-so
-that the borgmatic executable is available. For instance, adding this to
-root's `~/.profile` or `~/.bash_profile` may do the trick:
+This installs borgmatic and its commands at the `/root/.local/bin` path.
+
+Your pip binary may have a different name than "pip3". Make sure you're using
+Python 3, as borgmatic does not support Python 2.
+
+The next step is to ensure that borgmatic's commands available are on your
+system `PATH`, so that you can run borgmatic:
 
 ```bash
-export PATH="$PATH:~/.local/bin"
+echo export 'PATH="$PATH:/root/.local/bin"' >> ~/.bashrc
+source ~/.bashrc
 ```
 
-Note that your pip binary may have a different name than "pip3". Make sure
-you're using Python 3, as borgmatic does not support Python 2.
+This adds `/root/.local/bin` to your non-root user's system `PATH`.
+
+If you're using a command shell other than Bash, you may need to use different
+commands here.
+
+You can check whether all of this worked with:
+
+```bash
+sudo borgmatic --version
+```
+
+If borgmatic is properly installed, that should output your borgmatic version.
+
+
+### Global install option
+
+If you try the user site installation above, and have problems making
+borgmatic commands runnable on your system `PATH`, an alternate approach is to
+install borgmatic globally.
+
+The following uninstalls borgmatic, and then reinstalls it such that borgmatic
+commands are on the default system `PATH`:
+
+```bash
+sudo pip3 uninstall borgmatic
+sudo pip3 install --upgrade borgmatic
+```
+
+The main downside of a global install is that borgmatic is less cleanly
+separated from the rest of your Python software, and there's the theoretical
+possibility of libary conflicts. But if you're okay with that, for instance
+on a relatively dedicated system, then a global install can work out just
+fine.
+
 
 ### Other ways to install
 
-Along with the above process, you have several other options for installing
-borgmatic:
+Besides the approaches described above, there are several other options for
+installing borgmatic:
 
  * [Docker image with scheduled backups](https://hub.docker.com/r/b3vis/borgmatic/)
  * [Docker base image](https://hub.docker.com/r/monachus/borgmatic/)
@@ -170,16 +206,20 @@ good idea to test that borgmatic is working. So to run borgmatic and start a
 backup, you can invoke it like this:
 
 ```bash
-sudo borgmatic --verbosity 1
+sudo borgmatic --verbosity 1 --files
 ```
 
+(No borgmatic `--files` flag? It's only present in newer versions of
+borgmatic. So try leaving it out, or upgrade borgmatic!)
+
 By default, this will also prune any old backups as per the configured
 retention policy, and check backups for consistency problems due to things
 like file damage.
 
-The verbosity flag makes borgmatic list the files that it's archiving, which
-are those that are new or changed since the last backup. Eyeball the list and
-see if it matches your expectations based on the configuration.
+The verbosity flag makes borgmatic show the steps it's performing. And the
+files flag lists each file that's new or changed since the last backup.
+Eyeball the list and see if it matches your expectations based on the
+configuration.
 
 If you'd like to specify an alternate configuration file path, use the
 `--config` flag. See `borgmatic --help` for more information.
@@ -228,6 +268,7 @@ issues when reading files to backup. If that happens to you, you may be
 interested in an [unofficial work-around for Full Disk
 Access](https://projects.torsion.org/witten/borgmatic/issues/293).
 
+
 ## Colored output
 
 Borgmatic produces colored terminal output by default. It is disabled when a
@@ -236,6 +277,7 @@ non-interactive terminal is detected (like a cron job), or when you use the
 setting the environment variable `PY_COLORS=False`, or setting the `color`
 option to `false` in the `output` section of configuration.
 
+
 ## Troubleshooting
 
 ### "found character that cannot start any token" error

sample/systemd/borgmatic.service

@@ -19,6 +19,7 @@ Restart=no
 # doesn't support this (pre-240 or so), you may have to remove this option.
 LogRateLimitIntervalSec=0
 
-# Delay start to prevent backups running during boot.
+# Delay start to prevent backups running during boot. Note that systemd-inhibit requires dbus and
+# dbus-user-session to be installed.
 ExecStartPre=sleep 1m
 ExecStart=systemd-inhibit --who="borgmatic" --why="Prevent interrupting scheduled backup" /root/.local/bin/borgmatic --syslog-verbosity 1

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.5.3'
+VERSION = '1.5.6'
 
 
 setup(

tests/end-to-end/test_borgmatic.py

@@ -68,7 +68,7 @@ def test_borgmatic_command():
         extracted_config_path = os.path.join(extract_path, config_path)
         assert open(extracted_config_path).read() == open(config_path).read()
 
-        # Exercise the info flag.
+        # Exercise the info action.
         output = subprocess.check_output(
             'borgmatic --config {} info --json'.format(config_path).split(' ')
         ).decode(sys.stdout.encoding)

tests/end-to-end/test_database.py

@@ -5,12 +5,16 @@ import subprocess
 import sys
 import tempfile
 
+import pytest
 
-def write_configuration(config_path, repository_path, borgmatic_source_directory):
+
+def write_configuration(
+    config_path, repository_path, borgmatic_source_directory, postgresql_dump_format='custom'
+):
     '''
     Write out borgmatic configuration into a file at the config path. Set the options so as to work
     for testing. This includes injecting the given repository path, borgmatic source directory for
-    storing database dumps, and encryption passphrase.
+    storing database dumps, dump format (for PostgreSQL), and encryption passphrase.
     '''
     config = '''
 location:
@@ -29,6 +33,7 @@ hooks:
           hostname: postgresql
           username: postgres
           password: test
+          format: {}
         - name: all
           hostname: postgresql
           username: postgres
@@ -43,7 +48,7 @@ hooks:
           username: root
           password: test
 '''.format(
-        config_path, repository_path, borgmatic_source_directory
+        config_path, repository_path, borgmatic_source_directory, postgresql_dump_format
     )
 
     config_file = open(config_path, 'w')
@@ -67,7 +72,7 @@ def test_database_dump_and_restore():
             'borgmatic -v 2 --config {} init --encryption repokey'.format(config_path).split(' ')
         )
 
-        # Run borgmatic to generate a backup archive including a database dump
+        # Run borgmatic to generate a backup archive including a database dump.
         subprocess.check_call('borgmatic create --config {} -v 2'.format(config_path).split(' '))
 
         # Get the created archive name.
@@ -89,3 +94,71 @@ def test_database_dump_and_restore():
     finally:
         os.chdir(original_working_directory)
         shutil.rmtree(temporary_directory)
+
+
+def test_database_dump_and_restore_with_directory_format():
+    # Create a Borg repository.
+    temporary_directory = tempfile.mkdtemp()
+    repository_path = os.path.join(temporary_directory, 'test.borg')
+    borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic')
+
+    original_working_directory = os.getcwd()
+
+    try:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+        write_configuration(
+            config_path,
+            repository_path,
+            borgmatic_source_directory,
+            postgresql_dump_format='directory',
+        )
+
+        subprocess.check_call(
+            'borgmatic -v 2 --config {} init --encryption repokey'.format(config_path).split(' ')
+        )
+
+        # Run borgmatic to generate a backup archive including a database dump.
+        subprocess.check_call('borgmatic create --config {} -v 2'.format(config_path).split(' '))
+
+        # Restore the database from the archive.
+        subprocess.check_call(
+            'borgmatic --config {} restore --archive latest'.format(config_path).split(' ')
+        )
+    finally:
+        os.chdir(original_working_directory)
+        shutil.rmtree(temporary_directory)
+
+
+def test_database_dump_with_error_causes_borgmatic_to_exit():
+    # Create a Borg repository.
+    temporary_directory = tempfile.mkdtemp()
+    repository_path = os.path.join(temporary_directory, 'test.borg')
+    borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic')
+
+    original_working_directory = os.getcwd()
+
+    try:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+        write_configuration(config_path, repository_path, borgmatic_source_directory)
+
+        subprocess.check_call(
+            'borgmatic -v 2 --config {} init --encryption repokey'.format(config_path).split(' ')
+        )
+
+        # Run borgmatic with a config override such that the database dump fails.
+        with pytest.raises(subprocess.CalledProcessError):
+            subprocess.check_call(
+                [
+                    'borgmatic',
+                    'create',
+                    '--config',
+                    config_path,
+                    '-v',
+                    '2',
+                    '--override',
+                    "hooks.postgresql_databases=[{'name': 'nope'}]",
+                ]
+            )
+    finally:
+        os.chdir(original_working_directory)
+        shutil.rmtree(temporary_directory)

tests/integration/test_execute.py

@@ -26,7 +26,7 @@ def test_log_outputs_logs_each_line_separately():
         (hi_process, there_process),
         exclude_stdouts=(),
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
 
@@ -49,7 +49,7 @@ def test_log_outputs_skips_logs_for_process_with_none_stdout():
         (hi_process, there_process),
         exclude_stdouts=(),
         output_log_level=logging.INFO,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
 
@@ -63,10 +63,9 @@ def test_log_outputs_includes_error_output_in_exception():
 
     with pytest.raises(subprocess.CalledProcessError) as error:
         module.log_outputs(
-            (process,), exclude_stdouts=(), output_log_level=logging.INFO, error_on_warnings=False
+            (process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
         )
 
-    assert error.value.returncode == 2
     assert error.value.output
 
 
@@ -80,13 +79,42 @@ def test_log_outputs_skips_error_output_in_exception_for_process_with_none_stdou
 
     with pytest.raises(subprocess.CalledProcessError) as error:
         module.log_outputs(
-            (process,), exclude_stdouts=(), output_log_level=logging.INFO, error_on_warnings=False
+            (process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
         )
 
     assert error.value.returncode == 2
     assert not error.value.output
 
 
+def test_log_outputs_kills_other_processes_when_one_errors():
+    flexmock(module.logger).should_receive('log')
+    flexmock(module).should_receive('exit_code_indicates_error').and_return(True)
+    flexmock(module).should_receive('command_for_process').and_return('grep')
+
+    process = subprocess.Popen(['grep'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    other_process = subprocess.Popen(
+        ['watch', 'true'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+    )
+    flexmock(module).should_receive('output_buffer_for_process').with_args(process, ()).and_return(
+        process.stdout
+    )
+    flexmock(module).should_receive('output_buffer_for_process').with_args(
+        other_process, ()
+    ).and_return(other_process.stdout)
+    flexmock(other_process).should_receive('kill').once()
+
+    with pytest.raises(subprocess.CalledProcessError) as error:
+        module.log_outputs(
+            (process, other_process),
+            exclude_stdouts=(),
+            output_log_level=logging.INFO,
+            borg_local_path='borg',
+        )
+
+    assert error.value.returncode == 2
+    assert error.value.output
+
+
 def test_log_outputs_truncates_long_error_output():
     flexmock(module).ERROR_OUTPUT_MAX_LINE_COUNT = 0
     flexmock(module.logger).should_receive('log')
@@ -98,7 +126,7 @@ def test_log_outputs_truncates_long_error_output():
 
     with pytest.raises(subprocess.CalledProcessError) as error:
         module.log_outputs(
-            (process,), exclude_stdouts=(), output_log_level=logging.INFO, error_on_warnings=False
+            (process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
         )
 
     assert error.value.returncode == 2
@@ -113,5 +141,5 @@ def test_log_outputs_with_no_output_logs_nothing():
     flexmock(module).should_receive('output_buffer_for_process').and_return(process.stdout)
 
     module.log_outputs(
-        (process,), exclude_stdouts=(), output_log_level=logging.INFO, error_on_warnings=False
+        (process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
     )

tests/unit/borg/test_check.py

@@ -9,9 +9,7 @@ from ..test_verbosity import insert_logging_mock
 
 
 def insert_execute_command_mock(command):
-    flexmock(module).should_receive('execute_command').with_args(
-        command, error_on_warnings=True
-    ).once()
+    flexmock(module).should_receive('execute_command').with_args(command).once()
 
 
 def insert_execute_command_never():
@@ -165,9 +163,7 @@ def test_check_archives_with_progress_calls_borg_with_progress_parameter():
     flexmock(module).should_receive('_make_check_flags').and_return(())
     flexmock(module).should_receive('execute_command').never()
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'check', '--progress', 'repo'),
-        output_file=module.DO_NOT_CAPTURE,
-        error_on_warnings=True,
+        ('borg', 'check', '--progress', 'repo'), output_file=module.DO_NOT_CAPTURE
     ).once()
 
     module.check_archives(
@@ -182,9 +178,7 @@ def test_check_archives_with_repair_calls_borg_with_repair_parameter():
     flexmock(module).should_receive('_make_check_flags').and_return(())
     flexmock(module).should_receive('execute_command').never()
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'check', '--repair', 'repo'),
-        output_file=module.DO_NOT_CAPTURE,
-        error_on_warnings=True,
+        ('borg', 'check', '--repair', 'repo'), output_file=module.DO_NOT_CAPTURE
     ).once()
 
     module.check_archives(

tests/unit/borg/test_create.py

@@ -60,6 +60,26 @@ def test_expand_home_directories_considers_none_as_no_directories():
     assert paths == ()
 
 
+@pytest.mark.parametrize(
+    'directories,expected_directories',
+    (
+        (('/', '/root'), ('/',)),
+        (('/', '/root/'), ('/',)),
+        (('/root', '/'), ('/',)),
+        (('/root', '/root/foo'), ('/root',)),
+        (('/root/', '/root/foo'), ('/root/',)),
+        (('/root', '/root/foo/'), ('/root',)),
+        (('/root/foo', '/root'), ('/root',)),
+        (('/root', '/etc', '/root/foo/bar'), ('/etc', '/root')),
+        (('/root', '/root/foo', '/root/foo/bar'), ('/root',)),
+        (('/dup', '/dup'), ('/dup',)),
+        (('/foo', '/bar'), ('/bar', '/foo')),
+    ),
+)
+def test_deduplicate_directories_removes_child_paths(directories, expected_directories):
+    assert module.deduplicate_directories(directories) == expected_directories
+
+
 def test_write_pattern_file_does_not_raise():
     temporary_file = flexmock(name='filename', write=lambda mode: None, flush=lambda: None)
     flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file)
@@ -214,7 +234,8 @@ ARCHIVE_WITH_PATHS = ('repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'bar')
 
 def test_create_archive_calls_borg_with_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -223,7 +244,7 @@ def test_create_archive_calls_borg_with_parameters():
         ('borg', 'create') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -241,7 +262,8 @@ def test_create_archive_calls_borg_with_parameters():
 def test_create_archive_with_patterns_calls_borg_with_patterns():
     pattern_flags = ('--patterns-from', 'patterns')
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(
         flexmock(name='/tmp/patterns')
@@ -252,7 +274,7 @@ def test_create_archive_with_patterns_calls_borg_with_patterns():
         ('borg', 'create') + pattern_flags + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -270,7 +292,8 @@ def test_create_archive_with_patterns_calls_borg_with_patterns():
 def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
     exclude_flags = ('--exclude-from', 'excludes')
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(('exclude',))
     flexmock(module).should_receive('_write_pattern_file').and_return(None).and_return(
         flexmock(name='/tmp/excludes')
@@ -281,7 +304,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
         ('borg', 'create') + exclude_flags + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -298,7 +321,8 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
 
 def test_create_archive_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -308,7 +332,7 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
         ('borg', 'create', '--info') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.INFO)
 
@@ -326,7 +350,8 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
 
 def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -336,7 +361,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
         ('borg', 'create', '--json') + ARCHIVE_WITH_PATHS,
         output_log_level=None,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.INFO)
 
@@ -355,7 +380,8 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
 
 def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -364,7 +390,7 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
         ('borg', 'create', '--debug', '--show-rc') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -382,7 +408,8 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
 
 def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -391,7 +418,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
         ('borg', 'create', '--json') + ARCHIVE_WITH_PATHS,
         output_log_level=None,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -410,7 +437,8 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
 
 def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -420,7 +448,7 @@ def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
         ('borg', 'create', '--dry-run') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -439,7 +467,8 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete
     # --dry-run and --stats are mutually exclusive, see:
     # https://borgbackup.readthedocs.io/en/stable/usage/create.html#description
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -449,7 +478,7 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete
         ('borg', 'create', '--info', '--dry-run') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.INFO)
 
@@ -468,7 +497,8 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete
 
 def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_interval_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -477,7 +507,7 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
         ('borg', 'create', '--checkpoint-interval', '600') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -494,7 +524,8 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
 
 def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -503,7 +534,7 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param
         ('borg', 'create', '--chunker-params', '1,2,3,4') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -520,7 +551,8 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param
 
 def test_create_archive_with_compression_calls_borg_with_compression_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -529,7 +561,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
         ('borg', 'create', '--compression', 'rle') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -546,7 +578,8 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
 
 def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -555,7 +588,7 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
         ('borg', 'create', '--remote-ratelimit', '100') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -572,7 +605,8 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
 
 def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -581,7 +615,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
         ('borg', 'create', '--one-file-system') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -599,7 +633,8 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
 
 def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -608,7 +643,7 @@ def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_paramet
         ('borg', 'create', '--numeric-owner') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -626,7 +661,8 @@ def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_paramet
 
 def test_create_archive_with_read_special_calls_borg_with_read_special_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -635,7 +671,7 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
         ('borg', 'create', '--read-special') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -654,7 +690,8 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
 @pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags'))
 def test_create_archive_with_option_true_calls_borg_without_corresponding_parameter(option_name):
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -663,7 +700,7 @@ def test_create_archive_with_option_true_calls_borg_without_corresponding_parame
         ('borg', 'create') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -682,7 +719,8 @@ def test_create_archive_with_option_true_calls_borg_without_corresponding_parame
 @pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags'))
 def test_create_archive_with_option_false_calls_borg_with_corresponding_parameter(option_name):
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -691,7 +729,7 @@ def test_create_archive_with_option_false_calls_borg_with_corresponding_paramete
         ('borg', 'create', '--no' + option_name.replace('_', '')) + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -709,7 +747,8 @@ def test_create_archive_with_option_false_calls_borg_with_corresponding_paramete
 
 def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -718,7 +757,7 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
         ('borg', 'create', '--files-cache', 'ctime,size') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -736,7 +775,8 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
 
 def test_create_archive_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -745,7 +785,7 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
         ('borg1', 'create') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg1',
     )
 
     module.create_archive(
@@ -763,7 +803,8 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
 
 def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -772,7 +813,7 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
         ('borg', 'create', '--remote-path', 'borg1') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -790,7 +831,8 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
 
 def test_create_archive_with_umask_calls_borg_with_umask_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -799,7 +841,7 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
         ('borg', 'create', '--umask', '740') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -816,7 +858,8 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
 
 def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -825,7 +868,7 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
         ('borg', 'create', '--lock-wait', '5') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -842,7 +885,8 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
 
 def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_warning_output_log_level():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -851,7 +895,7 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_warning_o
         ('borg', 'create', '--stats') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.WARNING,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -869,7 +913,8 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_warning_o
 
 def test_create_archive_with_stats_and_log_info_calls_borg_with_stats_parameter_and_info_output_log_level():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -878,7 +923,7 @@ def test_create_archive_with_stats_and_log_info_calls_borg_with_stats_parameter_
         ('borg', 'create', '--info', '--stats') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.INFO)
 
@@ -897,7 +942,8 @@ def test_create_archive_with_stats_and_log_info_calls_borg_with_stats_parameter_
 
 def test_create_archive_with_files_calls_borg_with_list_parameter_and_warning_output_log_level():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -906,7 +952,7 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_warning_ou
         ('borg', 'create', '--list', '--filter', 'AME-') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.WARNING,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -924,7 +970,8 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_warning_ou
 
 def test_create_archive_with_files_and_log_info_calls_borg_with_list_parameter_and_info_output_log_level():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -933,7 +980,7 @@ def test_create_archive_with_files_and_log_info_calls_borg_with_list_parameter_a
         ('borg', 'create', '--list', '--filter', 'AME-', '--info') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.INFO)
 
@@ -952,7 +999,8 @@ def test_create_archive_with_files_and_log_info_calls_borg_with_list_parameter_a
 
 def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_parameter_and_no_list():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -961,7 +1009,7 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para
         ('borg', 'create', '--info', '--progress') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=module.DO_NOT_CAPTURE,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.INFO)
 
@@ -980,7 +1028,8 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para
 
 def test_create_archive_with_progress_calls_borg_with_progress_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -989,7 +1038,7 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter():
         ('borg', 'create', '--progress') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=module.DO_NOT_CAPTURE,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -1008,17 +1057,19 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter():
 def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progress_parameter():
     processes = flexmock()
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command_with_processes').with_args(
-        ('borg', 'create', '--read-special', '--progress') + ARCHIVE_WITH_PATHS,
+        ('borg', 'create', '--one-file-system', '--read-special', '--progress')
+        + ARCHIVE_WITH_PATHS,
         processes=processes,
         output_log_level=logging.INFO,
         output_file=module.DO_NOT_CAPTURE,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -1037,7 +1088,8 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr
 
 def test_create_archive_with_json_calls_borg_with_json_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1046,7 +1098,7 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
         ('borg', 'create', '--json') + ARCHIVE_WITH_PATHS,
         output_log_level=None,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     ).and_return('[]')
 
     json_output = module.create_archive(
@@ -1066,7 +1118,8 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
 
 def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1075,7 +1128,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter()
         ('borg', 'create', '--json') + ARCHIVE_WITH_PATHS,
         output_log_level=None,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     ).and_return('[]')
 
     json_output = module.create_archive(
@@ -1096,7 +1149,8 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter()
 
 def test_create_archive_with_source_directories_glob_expands():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'food'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1105,7 +1159,7 @@ def test_create_archive_with_source_directories_glob_expands():
         ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'),
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
 
@@ -1123,7 +1177,8 @@ def test_create_archive_with_source_directories_glob_expands():
 
 def test_create_archive_with_non_matching_source_directories_glob_passes_through():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo*',))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo*',))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1132,7 +1187,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
         ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo*'),
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
     flexmock(module.glob).should_receive('glob').with_args('foo*').and_return([])
 
@@ -1150,7 +1205,8 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
 
 def test_create_archive_with_glob_calls_borg_with_expanded_directories():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'food'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1159,7 +1215,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
         ('borg', 'create', 'repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'food'),
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -1176,7 +1232,8 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
 
 def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1185,7 +1242,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
         ('borg', 'create', 'repo::ARCHIVE_NAME', 'foo', 'bar'),
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -1202,7 +1259,8 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
 
 def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1211,7 +1269,7 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
         ('borg', 'create', 'repo::Documents_{hostname}-{now}', 'foo', 'bar'),
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -1228,7 +1286,8 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
 
 def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1237,7 +1296,7 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
         ('borg', 'create', '--extra', '--options') + ARCHIVE_WITH_PATHS,
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(
@@ -1255,17 +1314,18 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
 def test_create_archive_with_stream_processes_calls_borg_with_processes():
     processes = flexmock()
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command_with_processes').with_args(
-        ('borg', 'create', '--read-special') + ARCHIVE_WITH_PATHS,
+        ('borg', 'create', '--one-file-system', '--read-special') + ARCHIVE_WITH_PATHS,
         processes=processes,
         output_log_level=logging.INFO,
         output_file=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.create_archive(

tests/unit/borg/test_environment.py

@@ -60,3 +60,15 @@ def test_initialize_with_relocated_repo_access_should_override_default():
         assert os.environ.get('BORG_RELOCATED_REPO_ACCESS_IS_OK') == 'yes'
     finally:
         os.environ = orig_environ
+
+
+def test_initialize_is_not_affected_by_existing_environment():
+    orig_environ = os.environ
+
+    try:
+        os.environ = {'BORG_PASSPHRASE': 'pass', 'BORG_SSH': 'mosh'}
+        module.initialize({'ssh_command': 'ssh -C'})
+        assert 'BORG_PASSPHRASE' not in os.environ
+        assert os.environ.get('BORG_RSH') == 'ssh -C'
+    finally:
+        os.environ = orig_environ

tests/unit/borg/test_extract.py

@@ -8,15 +8,15 @@ from borgmatic.borg import extract as module
 from ..test_verbosity import insert_logging_mock
 
 
-def insert_execute_command_mock(command, working_directory=None, error_on_warnings=True):
+def insert_execute_command_mock(command, working_directory=None):
     flexmock(module).should_receive('execute_command').with_args(
-        command, working_directory=working_directory, error_on_warnings=error_on_warnings
+        command, working_directory=working_directory
     ).once()
 
 
 def insert_execute_command_output_mock(command, result):
     flexmock(module).should_receive('execute_command').with_args(
-        command, output_log_level=None, error_on_warnings=False
+        command, output_log_level=None, borg_local_path=command[0]
     ).and_return(result).once()
 
 
@@ -220,13 +220,27 @@ def test_extract_archive_calls_borg_with_destination_path():
     )
 
 
+def test_extract_archive_calls_borg_with_strip_components():
+    flexmock(module.os.path).should_receive('abspath').and_return('repo')
+    insert_execute_command_mock(('borg', 'extract', '--strip-components', '5', 'repo::archive'))
+
+    module.extract_archive(
+        dry_run=False,
+        repository='repo',
+        archive='archive',
+        paths=None,
+        location_config={},
+        storage_config={},
+        strip_components=5,
+    )
+
+
 def test_extract_archive_calls_borg_with_progress_parameter():
     flexmock(module.os.path).should_receive('abspath').and_return('repo')
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'extract', '--progress', 'repo::archive'),
         output_file=module.DO_NOT_CAPTURE,
         working_directory=None,
-        error_on_warnings=True,
     ).once()
 
     module.extract_archive(
@@ -263,7 +277,6 @@ def test_extract_archive_calls_borg_with_stdout_parameter_and_returns_process():
         ('borg', 'extract', '--stdout', 'repo::archive'),
         output_file=module.subprocess.PIPE,
         working_directory=None,
-        error_on_warnings=True,
         run_to_completion=False,
     ).and_return(process).once()
 
@@ -284,7 +297,7 @@ def test_extract_archive_calls_borg_with_stdout_parameter_and_returns_process():
 def test_extract_archive_skips_abspath_for_remote_repository():
     flexmock(module.os.path).should_receive('abspath').never()
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'extract', 'server:repo::archive'), working_directory=None, error_on_warnings=True
+        ('borg', 'extract', 'server:repo::archive'), working_directory=None
     ).once()
 
     module.extract_archive(

tests/unit/borg/test_info.py

@@ -10,7 +10,7 @@ from ..test_verbosity import insert_logging_mock
 
 def test_display_archives_info_calls_borg_with_parameters():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', 'repo'), output_log_level=logging.WARNING, error_on_warnings=False
+        ('borg', 'info', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg'
     )
 
     module.display_archives_info(
@@ -20,9 +20,7 @@ def test_display_archives_info_calls_borg_with_parameters():
 
 def test_display_archives_info_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', '--info', 'repo'),
-        output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        ('borg', 'info', '--info', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg'
     )
     insert_logging_mock(logging.INFO)
     module.display_archives_info(
@@ -32,7 +30,7 @@ def test_display_archives_info_with_log_info_calls_borg_with_info_parameter():
 
 def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', '--json', 'repo'), output_log_level=None, error_on_warnings=False
+        ('borg', 'info', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
     ).and_return('[]')
 
     insert_logging_mock(logging.INFO)
@@ -47,7 +45,7 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'info', '--debug', '--show-rc', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -58,7 +56,7 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter():
 
 def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', '--json', 'repo'), output_log_level=None, error_on_warnings=False
+        ('borg', 'info', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
     ).and_return('[]')
 
     insert_logging_mock(logging.DEBUG)
@@ -71,7 +69,7 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp
 
 def test_display_archives_info_with_json_calls_borg_with_json_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', '--json', 'repo'), output_log_level=None, error_on_warnings=False
+        ('borg', 'info', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
     ).and_return('[]')
 
     json_output = module.display_archives_info(
@@ -83,7 +81,7 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter():
 
 def test_display_archives_info_with_archive_calls_borg_with_archive_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', 'repo::archive'), output_log_level=logging.WARNING, error_on_warnings=False
+        ('borg', 'info', 'repo::archive'), output_log_level=logging.WARNING, borg_local_path='borg'
     )
 
     module.display_archives_info(
@@ -93,7 +91,7 @@ def test_display_archives_info_with_archive_calls_borg_with_archive_parameter():
 
 def test_display_archives_info_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg1', 'info', 'repo'), output_log_level=logging.WARNING, error_on_warnings=False
+        ('borg1', 'info', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg1'
     )
 
     module.display_archives_info(
@@ -108,7 +106,7 @@ def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_para
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'info', '--remote-path', 'borg1', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.display_archives_info(
@@ -124,7 +122,7 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'info', '--lock-wait', '5', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.display_archives_info(
@@ -139,7 +137,7 @@ def test_display_archives_info_passes_through_arguments_to_borg(argument_name):
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'info', '--' + argument_name.replace('_', '-'), 'value', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.display_archives_info(

tests/unit/borg/test_init.py

@@ -24,7 +24,7 @@ def insert_info_command_not_found_mock():
 
 def insert_init_command_mock(init_command, **kwargs):
     flexmock(module).should_receive('execute_command').with_args(
-        init_command, output_file=module.DO_NOT_CAPTURE, error_on_warnings=False
+        init_command, output_file=module.DO_NOT_CAPTURE, borg_local_path=init_command[0]
     ).once()
 
 

tests/unit/borg/test_list.py

@@ -26,9 +26,7 @@ def test_resolve_archive_name_passes_through_non_latest_archive_name():
 def test_resolve_archive_name_calls_borg_with_parameters():
     expected_archive = 'archive-name'
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
-        output_log_level=None,
-        error_on_warnings=False,
+        ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, output_log_level=None, borg_local_path='borg'
     ).and_return(expected_archive + '\n')
 
     assert module.resolve_archive_name('repo', 'latest', storage_config={}) == expected_archive
@@ -39,7 +37,7 @@ def test_resolve_archive_name_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--info') + BORG_LIST_LATEST_ARGUMENTS,
         output_log_level=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     ).and_return(expected_archive + '\n')
     insert_logging_mock(logging.INFO)
 
@@ -51,7 +49,7 @@ def test_resolve_archive_name_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--debug', '--show-rc') + BORG_LIST_LATEST_ARGUMENTS,
         output_log_level=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     ).and_return(expected_archive + '\n')
     insert_logging_mock(logging.DEBUG)
 
@@ -63,7 +61,7 @@ def test_resolve_archive_name_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg1', 'list') + BORG_LIST_LATEST_ARGUMENTS,
         output_log_level=None,
-        error_on_warnings=False,
+        borg_local_path='borg1',
     ).and_return(expected_archive + '\n')
 
     assert (
@@ -77,7 +75,7 @@ def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_param
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--remote-path', 'borg1') + BORG_LIST_LATEST_ARGUMENTS,
         output_log_level=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     ).and_return(expected_archive + '\n')
 
     assert (
@@ -88,9 +86,7 @@ def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_param
 
 def test_resolve_archive_name_without_archives_raises():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
-        output_log_level=None,
-        error_on_warnings=False,
+        ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, output_log_level=None, borg_local_path='borg'
     ).and_return('')
 
     with pytest.raises(ValueError):
@@ -103,7 +99,7 @@ def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_parameter
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--lock-wait', 'okay') + BORG_LIST_LATEST_ARGUMENTS,
         output_log_level=None,
-        error_on_warnings=False,
+        borg_local_path='borg',
     ).and_return(expected_archive + '\n')
 
     assert (
@@ -114,7 +110,7 @@ def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_parameter
 
 def test_list_archives_calls_borg_with_parameters():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', 'repo'), output_log_level=logging.WARNING, error_on_warnings=False
+        ('borg', 'list', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg'
     )
 
     module.list_archives(
@@ -126,9 +122,7 @@ def test_list_archives_calls_borg_with_parameters():
 
 def test_list_archives_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', '--info', 'repo'),
-        output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        ('borg', 'list', '--info', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg'
     )
     insert_logging_mock(logging.INFO)
 
@@ -141,7 +135,7 @@ def test_list_archives_with_log_info_calls_borg_with_info_parameter():
 
 def test_list_archives_with_log_info_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', '--json', 'repo'), output_log_level=None, error_on_warnings=False
+        ('borg', 'list', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
     )
     insert_logging_mock(logging.INFO)
 
@@ -156,7 +150,7 @@ def test_list_archives_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--debug', '--show-rc', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -169,7 +163,7 @@ def test_list_archives_with_log_debug_calls_borg_with_debug_parameter():
 
 def test_list_archives_with_log_debug_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', '--json', 'repo'), output_log_level=None, error_on_warnings=False
+        ('borg', 'list', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
     )
     insert_logging_mock(logging.DEBUG)
 
@@ -185,7 +179,7 @@ def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--lock-wait', '5', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.list_archives(
@@ -198,7 +192,7 @@ def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
 def test_list_archives_with_archive_calls_borg_with_archive_parameter():
     storage_config = {}
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', 'repo::archive'), output_log_level=logging.WARNING, error_on_warnings=False
+        ('borg', 'list', 'repo::archive'), output_log_level=logging.WARNING, borg_local_path='borg'
     )
 
     module.list_archives(
@@ -213,7 +207,7 @@ def test_list_archives_with_path_calls_borg_with_path_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', 'repo::archive', 'var/lib'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.list_archives(
@@ -225,7 +219,7 @@ def test_list_archives_with_path_calls_borg_with_path_parameter():
 
 def test_list_archives_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg1', 'list', 'repo'), output_log_level=logging.WARNING, error_on_warnings=False
+        ('borg1', 'list', 'repo'), output_log_level=logging.WARNING, borg_local_path='borg1'
     )
 
     module.list_archives(
@@ -240,7 +234,7 @@ def test_list_archives_with_remote_path_calls_borg_with_remote_path_parameters()
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--remote-path', 'borg1', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     )
 
     module.list_archives(
@@ -255,7 +249,7 @@ def test_list_archives_with_short_calls_borg_with_short_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--short', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     ).and_return('[]')
 
     module.list_archives(
@@ -283,7 +277,7 @@ def test_list_archives_passes_through_arguments_to_borg(argument_name):
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--' + argument_name.replace('_', '-'), 'value', 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     ).and_return('[]')
 
     module.list_archives(
@@ -299,7 +293,7 @@ def test_list_archives_with_successful_calls_borg_to_exclude_checkpoints():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'list', '--glob-archives', module.BORG_EXCLUDE_CHECKPOINTS_GLOB, 'repo'),
         output_log_level=logging.WARNING,
-        error_on_warnings=False,
+        borg_local_path='borg',
     ).and_return('[]')
 
     module.list_archives(
@@ -311,7 +305,7 @@ def test_list_archives_with_successful_calls_borg_to_exclude_checkpoints():
 
 def test_list_archives_with_json_calls_borg_with_json_parameter():
     flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', '--json', 'repo'), output_log_level=None, error_on_warnings=False
+        ('borg', 'list', '--json', 'repo'), output_log_level=None, borg_local_path='borg'
     ).and_return('[]')
 
     json_output = module.list_archives(

tests/unit/borg/test_mount.py

@@ -9,7 +9,7 @@ from ..test_verbosity import insert_logging_mock
 
 def insert_execute_command_mock(command):
     flexmock(module).should_receive('execute_command').with_args(
-        command, error_on_warnings=False
+        command, borg_local_path='borg'
     ).once()
 
 
@@ -120,7 +120,7 @@ def test_mount_archive_calls_borg_with_foreground_parameter():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg', 'mount', '--foreground', 'repo::archive', '/mnt'),
         output_file=module.DO_NOT_CAPTURE,
-        error_on_warnings=False,
+        borg_local_path='borg',
     ).once()
 
     module.mount_archive(

tests/unit/borg/test_prune.py

@@ -10,7 +10,7 @@ from ..test_verbosity import insert_logging_mock
 
 def insert_execute_command_mock(prune_command, output_log_level):
     flexmock(module).should_receive('execute_command').with_args(
-        prune_command, output_log_level=output_log_level, error_on_warnings=False
+        prune_command, output_log_level=output_log_level, borg_local_path=prune_command[0]
     ).once()
 
 

tests/unit/borg/test_umount.py

@@ -8,9 +8,7 @@ from ..test_verbosity import insert_logging_mock
 
 
 def insert_execute_command_mock(command):
-    flexmock(module).should_receive('execute_command').with_args(
-        command, error_on_warnings=True
-    ).once()
+    flexmock(module).should_receive('execute_command').with_args(command).once()
 
 
 def test_unmount_archive_calls_borg_with_required_parameters():

tests/unit/hooks/test_dump.py

@@ -34,60 +34,41 @@ def test_make_database_dump_filename_with_invalid_name_raises():
         module.make_database_dump_filename('databases', 'invalid/name')
 
 
-def test_create_named_pipe_for_dump_does_not_raise():
+def test_create_parent_directory_for_dump_does_not_raise():
     flexmock(module.os).should_receive('makedirs')
-    flexmock(module.os.path).should_receive('exists').and_return(False)
+
+    module.create_parent_directory_for_dump('/path/to/parent')
+
+
+def test_create_named_pipe_for_dump_does_not_raise():
+    flexmock(module).should_receive('create_parent_directory_for_dump')
     flexmock(module.os).should_receive('mkfifo')
 
     module.create_named_pipe_for_dump('/path/to/pipe')
 
 
-def test_remove_database_dumps_removes_dump_for_each_database():
-    databases = [{'name': 'foo'}, {'name': 'bar'}]
-    flexmock(module).should_receive('make_database_dump_filename').with_args(
-        'databases', 'foo', None
-    ).and_return('databases/localhost/foo')
-    flexmock(module).should_receive('make_database_dump_filename').with_args(
-        'databases', 'bar', None
-    ).and_return('databases/localhost/bar')
+def test_remove_database_dumps_removes_dump_path():
+    flexmock(module.os.path).should_receive('expanduser').and_return('databases/localhost')
+    flexmock(module.os.path).should_receive('exists').and_return(True)
+    flexmock(module.shutil).should_receive('rmtree').with_args('databases/localhost').once()
 
-    flexmock(module.os.path).should_receive('isdir').and_return(False)
-    flexmock(module.os).should_receive('remove').with_args('databases/localhost/foo').once()
-    flexmock(module.os).should_receive('remove').with_args('databases/localhost/bar').once()
-    flexmock(module.os).should_receive('listdir').with_args('databases/localhost').and_return(
-        ['bar']
-    ).and_return([])
-
-    flexmock(module.os).should_receive('rmdir').with_args('databases/localhost').once()
-
-    module.remove_database_dumps('databases', databases, 'SuperDB', 'test.yaml', dry_run=False)
-
-
-def test_remove_database_dumps_removes_dump_in_directory_format():
-    databases = [{'name': 'foo'}]
-    flexmock(module).should_receive('make_database_dump_filename').with_args(
-        'databases', 'foo', None
-    ).and_return('databases/localhost/foo')
-
-    flexmock(module.os.path).should_receive('isdir').and_return(True)
-    flexmock(module.os).should_receive('remove').never()
-    flexmock(module.shutil).should_receive('rmtree').with_args('databases/localhost/foo').once()
-    flexmock(module.os).should_receive('listdir').with_args('databases/localhost').and_return([])
-    flexmock(module.os).should_receive('rmdir').with_args('databases/localhost').once()
-
-    module.remove_database_dumps('databases', databases, 'SuperDB', 'test.yaml', dry_run=False)
+    module.remove_database_dumps('databases', 'SuperDB', 'test.yaml', dry_run=False)
 
 
 def test_remove_database_dumps_with_dry_run_skips_removal():
-    databases = [{'name': 'foo'}, {'name': 'bar'}]
-    flexmock(module.os).should_receive('rmdir').never()
-    flexmock(module.os).should_receive('remove').never()
+    flexmock(module.os.path).should_receive('expanduser').and_return('databases/localhost')
+    flexmock(module.os.path).should_receive('exists').never()
+    flexmock(module.shutil).should_receive('rmtree').never()
 
-    module.remove_database_dumps('databases', databases, 'SuperDB', 'test.yaml', dry_run=True)
+    module.remove_database_dumps('databases', 'SuperDB', 'test.yaml', dry_run=True)
 
 
-def test_remove_database_dumps_without_databases_does_not_raise():
-    module.remove_database_dumps('databases', [], 'SuperDB', 'test.yaml', dry_run=False)
+def test_remove_database_dumps_without_dump_path_present_skips_removal():
+    flexmock(module.os.path).should_receive('expanduser').and_return('databases/localhost')
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.shutil).should_receive('rmtree').never()
+
+    module.remove_database_dumps('databases', 'SuperDB', 'test.yaml', dry_run=False)
 
 
 def test_convert_glob_patterns_to_borg_patterns_removes_leading_slash():

tests/unit/hooks/test_mysql.py

@@ -198,6 +198,19 @@ def test_dump_databases_runs_mysqldump_for_all_databases():
     assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
 
 
+def test_dump_databases_errors_for_missing_all_databases():
+    databases = [{'name': 'all'}]
+    process = flexmock()
+    flexmock(module).should_receive('make_dump_path').and_return('')
+    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
+        'databases/localhost/all'
+    )
+    flexmock(module).should_receive('database_names_to_dump').and_return(())
+
+    with pytest.raises(ValueError):
+        assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
+
+
 def test_restore_database_dump_runs_mysql_to_restore():
     database_config = [{'name': 'foo'}]
     extract_process = flexmock(stdout=flexmock())
@@ -208,6 +221,7 @@ def test_restore_database_dump_runs_mysql_to_restore():
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
         extra_environment=None,
+        borg_local_path='borg',
     ).once()
 
     module.restore_database_dump(
@@ -247,6 +261,7 @@ def test_restore_database_dump_runs_mysql_with_hostname_and_port():
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
         extra_environment=None,
+        borg_local_path='borg',
     ).once()
 
     module.restore_database_dump(
@@ -264,6 +279,7 @@ def test_restore_database_dump_runs_mysql_with_username_and_password():
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
         extra_environment={'MYSQL_PWD': 'trustsome1'},
+        borg_local_path='borg',
     ).once()
 
     module.restore_database_dump(

tests/unit/hooks/test_postgresql.py

@@ -112,14 +112,15 @@ def test_dump_databases_runs_pg_dump_with_username_and_password():
     assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
 
 
-def test_dump_databases_runs_pg_dump_with_format():
-    databases = [{'name': 'foo', 'format': 'tar'}]
+def test_dump_databases_runs_pg_dump_with_directory_format():
+    databases = [{'name': 'foo', 'format': 'directory'}]
     process = flexmock()
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     )
-    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+    flexmock(module.dump).should_receive('create_parent_directory_for_dump')
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
 
     flexmock(module).should_receive('execute_command').with_args(
         (
@@ -128,10 +129,10 @@ def test_dump_databases_runs_pg_dump_with_format():
             '--clean',
             '--if-exists',
             '--format',
-            'tar',
-            'foo',
-            '>',
+            'directory',
+            '--file',
             'databases/localhost/foo',
+            'foo',
         ),
         shell=True,
         extra_environment=None,
@@ -194,6 +195,8 @@ def test_restore_database_dump_runs_pg_restore():
     database_config = [{'name': 'foo'}]
     extract_process = flexmock(stdout=flexmock())
 
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
     flexmock(module).should_receive('execute_command_with_processes').with_args(
         (
             'pg_restore',
@@ -208,6 +211,7 @@ def test_restore_database_dump_runs_pg_restore():
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
         extra_environment=None,
+        borg_local_path='borg',
     ).once()
     flexmock(module).should_receive('execute_command').with_args(
         ('psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'),
@@ -222,6 +226,8 @@ def test_restore_database_dump_runs_pg_restore():
 def test_restore_database_dump_errors_on_multiple_database_config():
     database_config = [{'name': 'foo'}, {'name': 'bar'}]
 
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
     flexmock(module).should_receive('execute_command_with_processes').never()
     flexmock(module).should_receive('execute_command').never()
 
@@ -235,6 +241,8 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
     database_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
     extract_process = flexmock(stdout=flexmock())
 
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
     flexmock(module).should_receive('execute_command_with_processes').with_args(
         (
             'pg_restore',
@@ -253,6 +261,7 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
         extra_environment=None,
+        borg_local_path='borg',
     ).once()
     flexmock(module).should_receive('execute_command').with_args(
         (
@@ -280,6 +289,8 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password():
     database_config = [{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1'}]
     extract_process = flexmock(stdout=flexmock())
 
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
     flexmock(module).should_receive('execute_command_with_processes').with_args(
         (
             'pg_restore',
@@ -296,6 +307,7 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password():
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
         extra_environment={'PGPASSWORD': 'trustsome1'},
+        borg_local_path='borg',
     ).once()
     flexmock(module).should_receive('execute_command').with_args(
         (
@@ -321,12 +333,15 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
     database_config = [{'name': 'all'}]
     extract_process = flexmock(stdout=flexmock())
 
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
     flexmock(module).should_receive('execute_command_with_processes').with_args(
         ('psql', '--no-password'),
         processes=[extract_process],
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
         extra_environment=None,
+        borg_local_path='borg',
     ).once()
     flexmock(module).should_receive('execute_command').with_args(
         ('psql', '--no-password', '--quiet', '--command', 'ANALYZE'), extra_environment=None
@@ -340,8 +355,42 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
 def test_restore_database_dump_with_dry_run_skips_restore():
     database_config = [{'name': 'foo'}]
 
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
     flexmock(module).should_receive('execute_command_with_processes').never()
 
     module.restore_database_dump(
         database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock()
     )
+
+
+def test_restore_database_dump_without_extract_process_restores_from_disk():
+    database_config = [{'name': 'foo'}]
+
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename').and_return('/dump/path')
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        (
+            'pg_restore',
+            '--no-password',
+            '--if-exists',
+            '--exit-on-error',
+            '--clean',
+            '--dbname',
+            'foo',
+            '/dump/path',
+        ),
+        processes=[],
+        output_log_level=logging.DEBUG,
+        input_file=None,
+        extra_environment=None,
+        borg_local_path='borg',
+    ).once()
+    flexmock(module).should_receive('execute_command').with_args(
+        ('psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'),
+        extra_environment=None,
+    ).once()
+
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=False, extract_process=None
+    )

tests/unit/test_execute.py

@@ -7,23 +7,27 @@ from borgmatic import execute as module
 
 
 @pytest.mark.parametrize(
-    'exit_code,error_on_warnings,expected_result',
+    'process,exit_code,borg_local_path,expected_result',
     (
-        (2, True, True),
-        (2, False, True),
-        (1, True, True),
-        (1, False, False),
-        (0, True, False),
-        (0, False, False),
+        (flexmock(args=['grep']), 2, None, True),
+        (flexmock(args=['grep']), 2, 'borg', True),
+        (flexmock(args=['borg']), 2, 'borg', True),
+        (flexmock(args=['borg1']), 2, 'borg1', True),
+        (flexmock(args=['grep']), 1, None, True),
+        (flexmock(args=['grep']), 1, 'borg', True),
+        (flexmock(args=['borg']), 1, 'borg', False),
+        (flexmock(args=['borg1']), 1, 'borg1', False),
+        (flexmock(args=['grep']), 0, None, False),
+        (flexmock(args=['grep']), 0, 'borg', False),
+        (flexmock(args=['borg']), 0, 'borg', False),
+        (flexmock(args=['borg1']), 0, 'borg1', False),
+        (flexmock(args=['borg']), None, None, False),
     ),
 )
-def test_exit_code_indicates_error_respects_exit_code_and_error_on_warnings(
-    exit_code, error_on_warnings, expected_result
+def test_exit_code_indicates_error_respects_exit_code_and_borg_local_path(
+    process, exit_code, borg_local_path, expected_result
 ):
-    assert (
-        module.exit_code_indicates_error(exit_code, error_on_warnings=error_on_warnings)
-        is expected_result
-    )
+    assert module.exit_code_indicates_error(process, exit_code, borg_local_path) is expected_result
 
 
 def test_command_for_process_converts_sequence_command_to_string():