Bump version for release.

NEWS

@@ -1,3 +1,14 @@
+1.5.5
+ * #314: Fix regression in support for PostgreSQL's "directory" dump format. Unlike other dump
+   formats, the "directory" dump format does not stream directly to/from Borg.
+ * #315: Fix enabled database hooks to implicitly set one_file_system configuration option to true.
+   This prevents Borg from reading devices like /dev/zero and hanging.
+ * #316: Fix hang when streaming a database dump to Borg with implicit duplicate source directories
+   by deduplicating them first.
+ * #319: Fix error message when there are no MySQL databases to dump for "all" databases.
+ * Improve documentation around the installation process. Specifically, making borgmatic commands
+   runnable via the system PATH and offering a global install option.
+
 1.5.4
  * #310: Fix legitimate database dump command errors (exit code 1) not being treated as errors by
    borgmatic.

borgmatic/borg/create.py

@@ -2,6 +2,7 @@ import glob
 import itertools
 import logging
 import os
+import pathlib
 import tempfile
 
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command, execute_command_with_processes
@@ -43,6 +44,35 @@ def _expand_home_directories(directories):
     return tuple(os.path.expanduser(directory) for directory in directories)
 
 
+def deduplicate_directories(directories):
+    '''
+    Given a sequence of directories, return them as a sorted tuple with all duplicate child
+    directories removed. For instance, if paths is ('/foo', '/foo/bar'), return just: ('/foo',)
+
+    The idea is that if Borg is given a parent directory, then it doesn't also need to be given
+    child directories, because it will naturally spider the contents of the parent directory. And
+    there are cases where Borg coming across the same file twice will result in duplicate reads and
+    even hangs, e.g. when a database hook is using a named pipe for streaming database dumps to
+    Borg.
+    '''
+    deduplicated = set()
+
+    for directory in sorted(directories):
+        # If the directory is "/", that contains all child directories, so we can early out.
+        if directory == os.path.sep:
+            return (os.path.sep,)
+
+        # If no other directories are parents of current directory (even n levels up), then the
+        # current directory isn't a duplicate.
+        if not any(
+            pathlib.PurePath(other_directory) in pathlib.PurePath(directory).parents
+            for other_directory in directories
+        ):
+            deduplicated.add(directory)
+
+    return tuple(sorted(deduplicated))
+
+
 def _write_pattern_file(patterns=None):
     '''
     Given a sequence of patterns, write them to a named temporary file and return it. Return None
@@ -148,9 +178,11 @@ def create_archive(
     If a sequence of stream processes is given (instances of subprocess.Popen), then execute the
     create command while also triggering the given processes to produce output.
     '''
-    sources = _expand_directories(
-        location_config['source_directories']
-        + borgmatic_source_directories(location_config.get('borgmatic_source_directory'))
+    sources = deduplicate_directories(
+        _expand_directories(
+            location_config['source_directories']
+            + borgmatic_source_directories(location_config.get('borgmatic_source_directory'))
+        )
     )
 
     pattern_file = _write_pattern_file(location_config.get('patterns'))
@@ -175,7 +207,11 @@ def create_archive(
         + (('--chunker-params', chunker_params) if chunker_params else ())
         + (('--compression', compression) if compression else ())
         + (('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ())
-        + (('--one-file-system',) if location_config.get('one_file_system') else ())
+        + (
+            ('--one-file-system',)
+            if location_config.get('one_file_system') or stream_processes
+            else ()
+        )
         + (('--numeric-owner',) if location_config.get('numeric_owner') else ())
         + (('--noatime',) if location_config.get('atime') is False else ())
         + (('--noctime',) if location_config.get('ctime') is False else ())

borgmatic/commands/borgmatic.py

@@ -254,6 +254,14 @@ def run_actions(
         )
     if 'create' in arguments:
         logger.info('{}: Creating archive{}'.format(repository, dry_run_label))
+        dispatch.call_hooks(
+            'remove_database_dumps',
+            hooks,
+            repository,
+            dump.DATABASE_HOOK_NAMES,
+            location,
+            global_arguments.dry_run,
+        )
         active_dumps = dispatch.call_hooks(
             'dump_databases',
             hooks,
@@ -346,6 +354,14 @@ def run_actions(
                     repository, arguments['restore'].archive
                 )
             )
+            dispatch.call_hooks(
+                'remove_database_dumps',
+                hooks,
+                repository,
+                dump.DATABASE_HOOK_NAMES,
+                location,
+                global_arguments.dry_run,
+            )
 
             restore_names = arguments['restore'].databases or []
             if 'all' in restore_names:
@@ -386,10 +402,12 @@ def run_actions(
                         local_path=local_path,
                         remote_path=remote_path,
                         destination_path='/',
-                        extract_to_stdout=True,
+                        # A directory format dump isn't a single file, and therefore can't extract
+                        # to stdout. In this case, the extract_process return value is None.
+                        extract_to_stdout=bool(restore_database.get('format') != 'directory'),
                     )
 
-                    # Run a single database restore, consuming the extract stdout.
+                    # Run a single database restore, consuming the extract stdout (if any).
                     dispatch.call_hooks(
                         'restore_database_dump',
                         {hook_name: [restore_database]},
@@ -400,6 +418,15 @@ def run_actions(
                         extract_process,
                     )
 
+            dispatch.call_hooks(
+                'remove_database_dumps',
+                hooks,
+                repository,
+                dump.DATABASE_HOOK_NAMES,
+                location,
+                global_arguments.dry_run,
+            )
+
             if not restore_names and not found_names:
                 raise ValueError('No databases were found to restore')
 

borgmatic/config/schema.yaml

@@ -30,7 +30,10 @@ map:
                     - user@backupserver:sourcehostname.borg
             one_file_system:
                 type: bool
-                desc: Stay in same file system (do not cross mount points). Defaults to false.
+                desc: |
+                  Stay in same file system (do not cross mount points). Defaults to false. But when
+                  a database hook is used, the setting here is ignored and one_file_system is
+                  considered true.
                 example: true
             numeric_owner:
                 type: bool
@@ -53,7 +56,9 @@ map:
                 desc: |
                     Use Borg's --read-special flag to allow backup of block and other special
                     devices. Use with caution, as it will lead to problems if used when
-                    backing up special devices such as /dev/zero. Defaults to false.
+                    backing up special devices such as /dev/zero. Defaults to false. But when a
+                    database hook is used, the setting here is ignored and read_special is
+                    considered true.
                 example: false
             bsd_flags:
                 type: bool
@@ -452,7 +457,9 @@ map:
                             type: str
                             desc: |
                                 Database name (required if using this hook). Or "all" to dump all
-                                databases on the host.
+                                databases on the host. Note that using this database hook implicitly
+                                enables both read_special and one_file_system (see above) to support
+                                dump and restore streaming.
                             example: users
                         hostname:
                             type: str
@@ -508,7 +515,9 @@ map:
                             type: str
                             desc: |
                                 Database name (required if using this hook). Or "all" to dump all
-                                databases on the host.
+                                databases on the host. Note that using this database hook implicitly
+                                enables both read_special and one_file_system (see above) to support
+                                dump and restore streaming.
                             example: users
                         hostname:
                             type: str

borgmatic/hooks/dump.py

@@ -33,14 +33,18 @@ def make_database_dump_filename(dump_path, name, hostname=None):
     return os.path.join(os.path.expanduser(dump_path), hostname or 'localhost', name)
 
 
+def create_parent_directory_for_dump(dump_path):
+    '''
+    Create a directory to contain the given dump path.
+    '''
+    os.makedirs(os.path.dirname(dump_path), mode=0o700, exist_ok=True)
+
+
 def create_named_pipe_for_dump(dump_path):
     '''
     Create a named pipe at the given dump path.
     '''
-    os.makedirs(os.path.dirname(dump_path), mode=0o700, exist_ok=True)
-    if os.path.exists(dump_path):
-        os.remove(dump_path)
-
+    create_parent_directory_for_dump(dump_path)
     os.mkfifo(dump_path, mode=0o600)
 
 
@@ -74,13 +78,15 @@ def remove_database_dumps(dump_path, databases, database_type_name, log_prefix,
         if dry_run:
             continue
 
-        if os.path.isdir(dump_filename):
-            shutil.rmtree(dump_filename)
-        else:
-            os.remove(dump_filename)
+        if os.path.exists(dump_filename):
+            if os.path.isdir(dump_filename):
+                shutil.rmtree(dump_filename)
+            else:
+                os.remove(dump_filename)
+
         dump_file_dir = os.path.dirname(dump_filename)
 
-        if len(os.listdir(dump_file_dir)) == 0:
+        if os.path.exists(dump_file_dir) and len(os.listdir(dump_file_dir)) == 0:
             os.rmdir(dump_file_dir)
 
 

borgmatic/hooks/mysql.py

@@ -73,9 +73,11 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
             make_dump_path(location_config), requested_name, database.get('hostname')
         )
         extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
-        dump_command_names = database_names_to_dump(
+        dump_database_names = database_names_to_dump(
             database, extra_environment, log_prefix, dry_run_label
         )
+        if not dump_database_names:
+            raise ValueError('Cannot find any MySQL databases to dump.')
 
         dump_command = (
             ('mysqldump',)
@@ -86,7 +88,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
             + (('--user', database['username']) if 'username' in database else ())
             + (tuple(database['options'].split(' ')) if 'options' in database else ())
             + ('--databases',)
-            + dump_command_names
+            + dump_database_names
             # Use shell redirection rather than execute_command(output_file=open(...)) to prevent
             # the open() call on a named pipe from hanging the main borgmatic process.
             + ('>', dump_filename)

borgmatic/hooks/postgresql.py

@@ -36,6 +36,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
             make_dump_path(location_config), name, database.get('hostname')
         )
         all_databases = bool(name == 'all')
+        dump_format = database.get('format', 'custom')
         command = (
             (
                 'pg_dumpall' if all_databases else 'pg_dump',
@@ -46,12 +47,14 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
             + (('--host', database['hostname']) if 'hostname' in database else ())
             + (('--port', str(database['port'])) if 'port' in database else ())
             + (('--username', database['username']) if 'username' in database else ())
-            + (() if all_databases else ('--format', database.get('format', 'custom')))
+            + (() if all_databases else ('--format', dump_format))
+            + (('--file', dump_filename) if dump_format == 'directory' else ())
             + (tuple(database['options'].split(' ')) if 'options' in database else ())
             + (() if all_databases else (name,))
             # Use shell redirection rather than the --file flag to sidestep synchronization issues
-            # when pg_dump/pg_dumpall tries to write to a named pipe.
-            + ('>', dump_filename)
+            # when pg_dump/pg_dumpall tries to write to a named pipe. But for the directory dump
+            # format in a particular, a named destination is required, and redirection doesn't work.
+            + (('>', dump_filename) if dump_format != 'directory' else ())
         )
         extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
 
@@ -63,7 +66,10 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
         if dry_run:
             continue
 
-        dump.create_named_pipe_for_dump(dump_filename)
+        if dump_format == 'directory':
+            dump.create_parent_directory_for_dump(dump_filename)
+        else:
+            dump.create_named_pipe_for_dump(dump_filename)
 
         processes.append(
             execute_command(
@@ -104,6 +110,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
     Use the given log prefix in any log entries. If this is a dry run, then don't actually restore
     anything. Trigger the given active extract process (an instance of subprocess.Popen) to produce
     output to consume.
+
+    If the extract process is None, then restore the dump from the filesystem rather than from an
+    extract stream.
     '''
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
 
@@ -112,6 +121,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
 
     database = database_config[0]
     all_databases = bool(database['name'] == 'all')
+    dump_filename = dump.make_database_dump_filename(
+        make_dump_path(location_config), database['name'], database.get('hostname')
+    )
     analyze_command = (
         ('psql', '--no-password', '--quiet')
         + (('--host', database['hostname']) if 'hostname' in database else ())
@@ -130,6 +142,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--username', database['username']) if 'username' in database else ())
+        + (() if extract_process else (dump_filename,))
     )
     extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
 
@@ -141,9 +154,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
 
     execute_command_with_processes(
         restore_command,
-        [extract_process],
+        [extract_process] if extract_process else [],
         output_log_level=logging.DEBUG,
-        input_file=extract_process.stdout,
+        input_file=extract_process.stdout if extract_process else None,
         extra_environment=extra_environment,
         borg_local_path=location_config.get('local_path', 'borg'),
     )

docs/how-to/backup-your-databases.md

@@ -24,12 +24,17 @@ hooks:
 
 As part of each backup, borgmatic streams a database dump for each configured
 database directly to Borg, so it's included in the backup without consuming
-additional disk space.
+additional disk space. (The one exception is PostgreSQL's "directory" dump
+format, which can't stream and therefore does consume temporary disk space.)
 
 To support this, borgmatic creates temporary named pipes in `~/.borgmatic` by
 default. To customize this path, set the `borgmatic_source_directory` option
 in the `location` section of borgmatic's configuration.
 
+Also note that using a database hook implicitly enables both the
+`read_special` and `one_file_system` configuration settings (even if they're
+disabled in your configuration) to support this dump and restore streaming.
+
 Here's a more involved example that connects to remote databases:
 
 ```yaml

docs/how-to/set-up-backups.md

@@ -12,30 +12,66 @@ First, [install
 Borg](https://borgbackup.readthedocs.io/en/stable/installation.html), at least
 version 1.1.
 
-Then, download and install borgmatic by running the following command:
+Then, download and install borgmatic as a [user site
+installation](https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site)
+by running the following command:
 
 ```bash
 sudo pip3 install --user --upgrade borgmatic
 ```
 
-This is a [recommended user site
-installation](https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site).
-You will need to ensure that `/root/.local/bin` is available on your `$PATH`
-so
-that the borgmatic executable is available. For instance, adding this to
-root's `~/.profile` or `~/.bash_profile` may do the trick:
+This installs borgmatic and its commands at the `/root/.local/bin` path.
+
+Your pip binary may have a different name than "pip3". Make sure you're using
+Python 3, as borgmatic does not support Python 2.
+
+The next step is to ensure that borgmatic's commands available are on your
+system `PATH`, so that you can run borgmatic:
 
 ```bash
-export PATH="$PATH:~/.local/bin"
+echo export 'PATH="$PATH:/root/.local/bin"' >> ~/.bashrc
+source ~/.bashrc
 ```
 
-Note that your pip binary may have a different name than "pip3". Make sure
-you're using Python 3, as borgmatic does not support Python 2.
+This adds `/root/.local/bin` to your non-root user's system `PATH`.
+
+If you're using a command shell other than Bash, you may need to use different
+commands here.
+
+You can check whether all of this worked with:
+
+```bash
+sudo borgmatic --version
+```
+
+If borgmatic is properly installed, that should output your borgmatic version.
+
+
+### Global install option
+
+If you try the user site installation above, and have problems making
+borgmatic commands runnable on your system `PATH`, an alternate approach is to
+install borgmatic globally.
+
+The following uninstalls borgmatic, and then reinstalls it such that borgmatic
+commands are on the default system `PATH`:
+
+```bash
+sudo pip3 uninstall borgmatic
+sudo pip3 install --upgrade borgmatic
+```
+
+The main downside of a global install is that borgmatic is less cleanly
+separated from the rest of your Python software, and there's the theoretical
+possibility of libary conflicts. But if you're okay with that, for instance
+on a relatively dedicated system, then a global install can work out just
+fine.
+
 
 ### Other ways to install
 
-Along with the above process, you have several other options for installing
-borgmatic:
+Besides the approaches described above, there are several other options for
+installing borgmatic:
 
  * [Docker image with scheduled backups](https://hub.docker.com/r/b3vis/borgmatic/)
  * [Docker base image](https://hub.docker.com/r/monachus/borgmatic/)
@@ -170,16 +206,20 @@ good idea to test that borgmatic is working. So to run borgmatic and start a
 backup, you can invoke it like this:
 
 ```bash
-sudo borgmatic --verbosity 1
+sudo borgmatic --verbosity 1 --files
 ```
 
+(No borgmatic `--files` flag? It's only present in newer versions of
+borgmatic. So try leaving it out, or upgrade borgmatic!)
+
 By default, this will also prune any old backups as per the configured
 retention policy, and check backups for consistency problems due to things
 like file damage.
 
-The verbosity flag makes borgmatic list the files that it's archiving, which
-are those that are new or changed since the last backup. Eyeball the list and
-see if it matches your expectations based on the configuration.
+The verbosity flag makes borgmatic show the steps it's performing. And the
+files flag lists each file that's new or changed since the last backup.
+Eyeball the list and see if it matches your expectations based on the
+configuration.
 
 If you'd like to specify an alternate configuration file path, use the
 `--config` flag. See `borgmatic --help` for more information.

sample/systemd/borgmatic.service

@@ -19,6 +19,7 @@ Restart=no
 # doesn't support this (pre-240 or so), you may have to remove this option.
 LogRateLimitIntervalSec=0
 
-# Delay start to prevent backups running during boot.
+# Delay start to prevent backups running during boot. Note that systemd-inhibit requires dbus and
+# dbus-user-session to be installed.
 ExecStartPre=sleep 1m
 ExecStart=systemd-inhibit --who="borgmatic" --why="Prevent interrupting scheduled backup" /root/.local/bin/borgmatic --syslog-verbosity 1

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.5.4'
+VERSION = '1.5.5'
 
 
 setup(

tests/end-to-end/test_database.py

@@ -8,11 +8,13 @@ import tempfile
 import pytest
 
 
-def write_configuration(config_path, repository_path, borgmatic_source_directory):
+def write_configuration(
+    config_path, repository_path, borgmatic_source_directory, postgresql_dump_format='custom'
+):
     '''
     Write out borgmatic configuration into a file at the config path. Set the options so as to work
     for testing. This includes injecting the given repository path, borgmatic source directory for
-    storing database dumps, and encryption passphrase.
+    storing database dumps, dump format (for PostgreSQL), and encryption passphrase.
     '''
     config = '''
 location:
@@ -31,6 +33,7 @@ hooks:
           hostname: postgresql
           username: postgres
           password: test
+          format: {}
         - name: all
           hostname: postgresql
           username: postgres
@@ -45,7 +48,7 @@ hooks:
           username: root
           password: test
 '''.format(
-        config_path, repository_path, borgmatic_source_directory
+        config_path, repository_path, borgmatic_source_directory, postgresql_dump_format
     )
 
     config_file = open(config_path, 'w')
@@ -93,6 +96,39 @@ def test_database_dump_and_restore():
         shutil.rmtree(temporary_directory)
 
 
+def test_database_dump_and_restore_with_directory_format():
+    # Create a Borg repository.
+    temporary_directory = tempfile.mkdtemp()
+    repository_path = os.path.join(temporary_directory, 'test.borg')
+    borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic')
+
+    original_working_directory = os.getcwd()
+
+    try:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+        write_configuration(
+            config_path,
+            repository_path,
+            borgmatic_source_directory,
+            postgresql_dump_format='directory',
+        )
+
+        subprocess.check_call(
+            'borgmatic -v 2 --config {} init --encryption repokey'.format(config_path).split(' ')
+        )
+
+        # Run borgmatic to generate a backup archive including a database dump.
+        subprocess.check_call('borgmatic create --config {} -v 2'.format(config_path).split(' '))
+
+        # Restore the database from the archive.
+        subprocess.check_call(
+            'borgmatic --config {} restore --archive latest'.format(config_path).split(' ')
+        )
+    finally:
+        os.chdir(original_working_directory)
+        shutil.rmtree(temporary_directory)
+
+
 def test_database_dump_with_error_causes_borgmatic_to_exit():
     # Create a Borg repository.
     temporary_directory = tempfile.mkdtemp()

tests/integration/test_execute.py

@@ -66,7 +66,6 @@ def test_log_outputs_includes_error_output_in_exception():
             (process,), exclude_stdouts=(), output_log_level=logging.INFO, borg_local_path='borg'
         )
 
-    assert error.value.returncode == 2
     assert error.value.output
 
 

tests/unit/borg/test_create.py

@@ -60,6 +60,26 @@ def test_expand_home_directories_considers_none_as_no_directories():
     assert paths == ()
 
 
+@pytest.mark.parametrize(
+    'directories,expected_directories',
+    (
+        (('/', '/root'), ('/',)),
+        (('/', '/root/'), ('/',)),
+        (('/root', '/'), ('/',)),
+        (('/root', '/root/foo'), ('/root',)),
+        (('/root/', '/root/foo'), ('/root/',)),
+        (('/root', '/root/foo/'), ('/root',)),
+        (('/root/foo', '/root'), ('/root',)),
+        (('/root', '/etc', '/root/foo/bar'), ('/etc', '/root')),
+        (('/root', '/root/foo', '/root/foo/bar'), ('/root',)),
+        (('/dup', '/dup'), ('/dup',)),
+        (('/foo', '/bar'), ('/bar', '/foo')),
+    ),
+)
+def test_deduplicate_directories_removes_child_paths(directories, expected_directories):
+    assert module.deduplicate_directories(directories) == expected_directories
+
+
 def test_write_pattern_file_does_not_raise():
     temporary_file = flexmock(name='filename', write=lambda mode: None, flush=lambda: None)
     flexmock(module.tempfile).should_receive('NamedTemporaryFile').and_return(temporary_file)
@@ -214,7 +234,8 @@ ARCHIVE_WITH_PATHS = ('repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'bar')
 
 def test_create_archive_calls_borg_with_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -241,7 +262,8 @@ def test_create_archive_calls_borg_with_parameters():
 def test_create_archive_with_patterns_calls_borg_with_patterns():
     pattern_flags = ('--patterns-from', 'patterns')
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(
         flexmock(name='/tmp/patterns')
@@ -270,7 +292,8 @@ def test_create_archive_with_patterns_calls_borg_with_patterns():
 def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
     exclude_flags = ('--exclude-from', 'excludes')
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(('exclude',))
     flexmock(module).should_receive('_write_pattern_file').and_return(None).and_return(
         flexmock(name='/tmp/excludes')
@@ -298,7 +321,8 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
 
 def test_create_archive_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -326,7 +350,8 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
 
 def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -355,7 +380,8 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
 
 def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -382,7 +408,8 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
 
 def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -410,7 +437,8 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
 
 def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -439,7 +467,8 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete
     # --dry-run and --stats are mutually exclusive, see:
     # https://borgbackup.readthedocs.io/en/stable/usage/create.html#description
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -468,7 +497,8 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete
 
 def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_interval_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -494,7 +524,8 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
 
 def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -520,7 +551,8 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param
 
 def test_create_archive_with_compression_calls_borg_with_compression_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -546,7 +578,8 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
 
 def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -572,7 +605,8 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
 
 def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -599,7 +633,8 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
 
 def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -626,7 +661,8 @@ def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_paramet
 
 def test_create_archive_with_read_special_calls_borg_with_read_special_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -654,7 +690,8 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
 @pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags'))
 def test_create_archive_with_option_true_calls_borg_without_corresponding_parameter(option_name):
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -682,7 +719,8 @@ def test_create_archive_with_option_true_calls_borg_without_corresponding_parame
 @pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags'))
 def test_create_archive_with_option_false_calls_borg_with_corresponding_parameter(option_name):
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -709,7 +747,8 @@ def test_create_archive_with_option_false_calls_borg_with_corresponding_paramete
 
 def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -736,7 +775,8 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
 
 def test_create_archive_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -763,7 +803,8 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
 
 def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -790,7 +831,8 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
 
 def test_create_archive_with_umask_calls_borg_with_umask_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -816,7 +858,8 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
 
 def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -842,7 +885,8 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
 
 def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_warning_output_log_level():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -869,7 +913,8 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_warning_o
 
 def test_create_archive_with_stats_and_log_info_calls_borg_with_stats_parameter_and_info_output_log_level():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -897,7 +942,8 @@ def test_create_archive_with_stats_and_log_info_calls_borg_with_stats_parameter_
 
 def test_create_archive_with_files_calls_borg_with_list_parameter_and_warning_output_log_level():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -924,7 +970,8 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_warning_ou
 
 def test_create_archive_with_files_and_log_info_calls_borg_with_list_parameter_and_info_output_log_level():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -952,7 +999,8 @@ def test_create_archive_with_files_and_log_info_calls_borg_with_list_parameter_a
 
 def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_parameter_and_no_list():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -980,7 +1028,8 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para
 
 def test_create_archive_with_progress_calls_borg_with_progress_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1008,13 +1057,15 @@ def test_create_archive_with_progress_calls_borg_with_progress_parameter():
 def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progress_parameter():
     processes = flexmock()
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command_with_processes').with_args(
-        ('borg', 'create', '--read-special', '--progress') + ARCHIVE_WITH_PATHS,
+        ('borg', 'create', '--one-file-system', '--read-special', '--progress')
+        + ARCHIVE_WITH_PATHS,
         processes=processes,
         output_log_level=logging.INFO,
         output_file=module.DO_NOT_CAPTURE,
@@ -1037,7 +1088,8 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr
 
 def test_create_archive_with_json_calls_borg_with_json_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1066,7 +1118,8 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
 
 def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1096,7 +1149,8 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter()
 
 def test_create_archive_with_source_directories_glob_expands():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'food'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1123,7 +1177,8 @@ def test_create_archive_with_source_directories_glob_expands():
 
 def test_create_archive_with_non_matching_source_directories_glob_passes_through():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo*',))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo*',))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1150,7 +1205,8 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
 
 def test_create_archive_with_glob_calls_borg_with_expanded_directories():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'food'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'food'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1176,7 +1232,8 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
 
 def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1202,7 +1259,8 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
 
 def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1228,7 +1286,8 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
 
 def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
@@ -1255,13 +1314,14 @@ def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
 def test_create_archive_with_stream_processes_calls_borg_with_processes():
     processes = flexmock()
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
-    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command_with_processes').with_args(
-        ('borg', 'create', '--read-special') + ARCHIVE_WITH_PATHS,
+        ('borg', 'create', '--one-file-system', '--read-special') + ARCHIVE_WITH_PATHS,
         processes=processes,
         output_log_level=logging.INFO,
         output_file=None,

tests/unit/hooks/test_dump.py

@@ -34,10 +34,14 @@ def test_make_database_dump_filename_with_invalid_name_raises():
         module.make_database_dump_filename('databases', 'invalid/name')
 
 
-def test_create_named_pipe_for_dump_does_not_raise():
+def test_create_parent_directory_for_dump_does_not_raise():
     flexmock(module.os).should_receive('makedirs')
-    flexmock(module.os.path).should_receive('exists').and_return(True)
-    flexmock(module.os).should_receive('remove')
+
+    module.create_parent_directory_for_dump('/path/to/parent')
+
+
+def test_create_named_pipe_for_dump_does_not_raise():
+    flexmock(module).should_receive('create_parent_directory_for_dump')
     flexmock(module.os).should_receive('mkfifo')
 
     module.create_named_pipe_for_dump('/path/to/pipe')
@@ -52,6 +56,7 @@ def test_remove_database_dumps_removes_dump_for_each_database():
         'databases', 'bar', None
     ).and_return('databases/localhost/bar')
 
+    flexmock(module.os.path).should_receive('exists').and_return(True)
     flexmock(module.os.path).should_receive('isdir').and_return(False)
     flexmock(module.os).should_receive('remove').with_args('databases/localhost/foo').once()
     flexmock(module.os).should_receive('remove').with_args('databases/localhost/bar').once()
@@ -70,6 +75,7 @@ def test_remove_database_dumps_removes_dump_in_directory_format():
         'databases', 'foo', None
     ).and_return('databases/localhost/foo')
 
+    flexmock(module.os.path).should_receive('exists').and_return(True)
     flexmock(module.os.path).should_receive('isdir').and_return(True)
     flexmock(module.os).should_receive('remove').never()
     flexmock(module.shutil).should_receive('rmtree').with_args('databases/localhost/foo').once()
@@ -87,6 +93,21 @@ def test_remove_database_dumps_with_dry_run_skips_removal():
     module.remove_database_dumps('databases', databases, 'SuperDB', 'test.yaml', dry_run=True)
 
 
+def test_remove_database_dumps_without_dump_present_skips_removal():
+    databases = [{'name': 'foo'}]
+    flexmock(module).should_receive('make_database_dump_filename').with_args(
+        'databases', 'foo', None
+    ).and_return('databases/localhost/foo')
+
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.os.path).should_receive('isdir').never()
+    flexmock(module.os).should_receive('remove').never()
+    flexmock(module.shutil).should_receive('rmtree').never()
+    flexmock(module.os).should_receive('rmdir').never()
+
+    module.remove_database_dumps('databases', databases, 'SuperDB', 'test.yaml', dry_run=False)
+
+
 def test_remove_database_dumps_without_databases_does_not_raise():
     module.remove_database_dumps('databases', [], 'SuperDB', 'test.yaml', dry_run=False)
 

tests/unit/hooks/test_mysql.py

@@ -198,6 +198,19 @@ def test_dump_databases_runs_mysqldump_for_all_databases():
     assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
 
 
+def test_dump_databases_errors_for_missing_all_databases():
+    databases = [{'name': 'all'}]
+    process = flexmock()
+    flexmock(module).should_receive('make_dump_path').and_return('')
+    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
+        'databases/localhost/all'
+    )
+    flexmock(module).should_receive('database_names_to_dump').and_return(())
+
+    with pytest.raises(ValueError):
+        assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
+
+
 def test_restore_database_dump_runs_mysql_to_restore():
     database_config = [{'name': 'foo'}]
     extract_process = flexmock(stdout=flexmock())

tests/unit/hooks/test_postgresql.py

@@ -112,14 +112,15 @@ def test_dump_databases_runs_pg_dump_with_username_and_password():
     assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
 
 
-def test_dump_databases_runs_pg_dump_with_format():
-    databases = [{'name': 'foo', 'format': 'tar'}]
+def test_dump_databases_runs_pg_dump_with_directory_format():
+    databases = [{'name': 'foo', 'format': 'directory'}]
     process = flexmock()
     flexmock(module).should_receive('make_dump_path').and_return('')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     )
-    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+    flexmock(module.dump).should_receive('create_parent_directory_for_dump')
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
 
     flexmock(module).should_receive('execute_command').with_args(
         (
@@ -128,10 +129,10 @@ def test_dump_databases_runs_pg_dump_with_format():
             '--clean',
             '--if-exists',
             '--format',
-            'tar',
-            'foo',
-            '>',
+            'directory',
+            '--file',
             'databases/localhost/foo',
+            'foo',
         ),
         shell=True,
         extra_environment=None,
@@ -194,6 +195,8 @@ def test_restore_database_dump_runs_pg_restore():
     database_config = [{'name': 'foo'}]
     extract_process = flexmock(stdout=flexmock())
 
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
     flexmock(module).should_receive('execute_command_with_processes').with_args(
         (
             'pg_restore',
@@ -223,6 +226,8 @@ def test_restore_database_dump_runs_pg_restore():
 def test_restore_database_dump_errors_on_multiple_database_config():
     database_config = [{'name': 'foo'}, {'name': 'bar'}]
 
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
     flexmock(module).should_receive('execute_command_with_processes').never()
     flexmock(module).should_receive('execute_command').never()
 
@@ -236,6 +241,8 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
     database_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
     extract_process = flexmock(stdout=flexmock())
 
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
     flexmock(module).should_receive('execute_command_with_processes').with_args(
         (
             'pg_restore',
@@ -282,6 +289,8 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password():
     database_config = [{'name': 'foo', 'username': 'postgres', 'password': 'trustsome1'}]
     extract_process = flexmock(stdout=flexmock())
 
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
     flexmock(module).should_receive('execute_command_with_processes').with_args(
         (
             'pg_restore',
@@ -324,6 +333,8 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
     database_config = [{'name': 'all'}]
     extract_process = flexmock(stdout=flexmock())
 
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
     flexmock(module).should_receive('execute_command_with_processes').with_args(
         ('psql', '--no-password'),
         processes=[extract_process],
@@ -344,8 +355,42 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
 def test_restore_database_dump_with_dry_run_skips_restore():
     database_config = [{'name': 'foo'}]
 
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
     flexmock(module).should_receive('execute_command_with_processes').never()
 
     module.restore_database_dump(
         database_config, 'test.yaml', {}, dry_run=True, extract_process=flexmock()
     )
+
+
+def test_restore_database_dump_without_extract_process_restores_from_disk():
+    database_config = [{'name': 'foo'}]
+
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename').and_return('/dump/path')
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        (
+            'pg_restore',
+            '--no-password',
+            '--if-exists',
+            '--exit-on-error',
+            '--clean',
+            '--dbname',
+            'foo',
+            '/dump/path',
+        ),
+        processes=[],
+        output_log_level=logging.DEBUG,
+        input_file=None,
+        extra_environment=None,
+        borg_local_path='borg',
+    ).once()
+    flexmock(module).should_receive('execute_command').with_args(
+        ('psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'),
+        extra_environment=None,
+    ).once()
+
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=False, extract_process=None
+    )