Bump version for release.

Merge pull request #49 from jpaniagualaconich/specify-pg-dump-restore-commands

NEWS

@@ -1,3 +1,15 @@
+1.7.5
+ * #311: Override PostgreSQL dump/restore commands via configuration options.
+ * #604: Fix traceback when a configuration section is present but lacking any options.
+ * #607: Clarify documentation examples for include merging and deep merging.
+ * #611: Fix "data" consistency check to support "check_last" and consistency "prefix" options.
+ * #613: Clarify documentation about multiple repositories and separate configuration files.
+
+1.7.4
+ * #596: Fix special file detection erroring when broken symlinks are encountered.
+ * #597, #598: Fix regression in which "check" action errored on certain systems ("Cannot determine
+   Borg repository ID").
+
 1.7.3
  * #357: Add "break-lock" action for removing any repository and cache locks leftover from Borg
    aborting.

borgmatic/borg/check.py

@@ -166,6 +166,12 @@ def make_check_flags(local_borg_version, checks, check_last=None, prefix=None):
     "--last" flag. And if a prefix value is given and "archives" is in checks, then include a
     "--match-archives" flag.
     '''
+    if 'data' in checks:
+        data_flags = ('--verify-data',)
+        checks += ('archives',)
+    else:
+        data_flags = ()
+
     if 'archives' in checks:
         last_flags = ('--last', str(check_last)) if check_last else ()
         if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
@@ -176,17 +182,13 @@ def make_check_flags(local_borg_version, checks, check_last=None, prefix=None):
         last_flags = ()
         match_archives_flags = ()
         if check_last:
-            logger.info('Ignoring check_last option, as "archives" is not in consistency checks')
+            logger.warning(
+                'Ignoring check_last option, as "archives" or "data" are not in consistency checks'
+            )
         if prefix:
-            logger.info(
-                'Ignoring consistency prefix option, as "archives" is not in consistency checks'
+            logger.warning(
+                'Ignoring consistency prefix option, as "archives" or "data" are not in consistency checks'
             )
-
-    if 'data' in checks:
-        data_flags = ('--verify-data',)
-        checks += ('archives',)
-    else:
-        data_flags = ()
 
     common_flags = last_flags + match_archives_flags + data_flags
 

borgmatic/borg/create.py

@@ -7,7 +7,12 @@ import stat
 import tempfile
 
 from borgmatic.borg import environment, feature, flags, state
-from borgmatic.execute import DO_NOT_CAPTURE, execute_command, execute_command_with_processes
+from borgmatic.execute import (
+    DO_NOT_CAPTURE,
+    execute_command,
+    execute_command_and_capture_output,
+    execute_command_with_processes,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -229,7 +234,11 @@ def special_file(path):
     Return whether the given path is a special file (character device, block device, or named pipe
     / FIFO).
     '''
-    mode = os.stat(path).st_mode
+    try:
+        mode = os.stat(path).st_mode
+    except (FileNotFoundError, OSError):
+        return False
+
     return stat.S_ISCHR(mode) or stat.S_ISBLK(mode) or stat.S_ISFIFO(mode)
 
 
@@ -255,10 +264,9 @@ def collect_special_file_paths(
     Borg would encounter during a create. These are all paths that could cause Borg to hang if its
     --read-special flag is used.
     '''
-    paths_output = execute_command(
+    paths_output = execute_command_and_capture_output(
         create_command + ('--dry-run', '--list'),
-        output_log_level=None,
-        borg_local_path=local_path,
+        capture_stderr=True,
         working_directory=working_directory,
         extra_environment=borg_environment,
     )
@@ -452,12 +460,16 @@ def create_archive(
             working_directory=working_directory,
             extra_environment=borg_environment,
         )
-
-    return execute_command(
-        create_command,
-        output_log_level,
-        output_file,
-        borg_local_path=local_path,
-        working_directory=working_directory,
-        extra_environment=borg_environment,
-    )
+    elif output_log_level is None:
+        return execute_command_and_capture_output(
+            create_command, working_directory=working_directory, extra_environment=borg_environment,
+        )
+    else:
+        execute_command(
+            create_command,
+            output_log_level,
+            output_file,
+            borg_local_path=local_path,
+            working_directory=working_directory,
+            extra_environment=borg_environment,
+        )

borgmatic/borg/info.py

@@ -1,7 +1,7 @@
 import logging
 
 from borgmatic.borg import environment, feature, flags
-from borgmatic.execute import execute_command
+from borgmatic.execute import execute_command, execute_command_and_capture_output
 
 logger = logging.getLogger(__name__)
 
@@ -55,9 +55,14 @@ def display_archives_info(
         )
     )
 
-    return execute_command(
-        full_command,
-        output_log_level=None if info_arguments.json else logging.WARNING,
-        borg_local_path=local_path,
-        extra_environment=environment.make_environment(storage_config),
-    )
+    if info_arguments.json:
+        return execute_command_and_capture_output(
+            full_command, extra_environment=environment.make_environment(storage_config),
+        )
+    else:
+        execute_command(
+            full_command,
+            output_log_level=logging.WARNING,
+            borg_local_path=local_path,
+            extra_environment=environment.make_environment(storage_config),
+        )

borgmatic/borg/list.py

@@ -4,7 +4,7 @@ import logging
 import re
 
 from borgmatic.borg import environment, feature, flags, rlist
-from borgmatic.execute import execute_command
+from borgmatic.execute import execute_command, execute_command_and_capture_output
 
 logger = logging.getLogger(__name__)
 
@@ -151,7 +151,7 @@ def list_archive(
 
         # Ask Borg to list archives. Capture its output for use below.
         archive_lines = tuple(
-            execute_command(
+            execute_command_and_capture_output(
                 rlist.make_rlist_command(
                     repository,
                     storage_config,
@@ -160,8 +160,6 @@ def list_archive(
                     local_path,
                     remote_path,
                 ),
-                output_log_level=None,
-                borg_local_path=local_path,
                 extra_environment=borg_environment,
             )
             .strip('\n')

borgmatic/borg/rinfo.py

@@ -1,7 +1,7 @@
 import logging
 
 from borgmatic.borg import environment, feature, flags
-from borgmatic.execute import execute_command
+from borgmatic.execute import execute_command, execute_command_and_capture_output
 
 logger = logging.getLogger(__name__)
 
@@ -44,9 +44,16 @@ def display_repository_info(
         + flags.make_repository_flags(repository, local_borg_version)
     )
 
-    return execute_command(
-        full_command,
-        output_log_level=None if rinfo_arguments.json else logging.WARNING,
-        borg_local_path=local_path,
-        extra_environment=environment.make_environment(storage_config),
-    )
+    extra_environment = environment.make_environment(storage_config)
+
+    if rinfo_arguments.json:
+        return execute_command_and_capture_output(
+            full_command, extra_environment=extra_environment,
+        )
+    else:
+        execute_command(
+            full_command,
+            output_log_level=logging.WARNING,
+            borg_local_path=local_path,
+            extra_environment=extra_environment,
+        )

borgmatic/borg/rlist.py

@@ -1,7 +1,7 @@
 import logging
 
 from borgmatic.borg import environment, feature, flags
-from borgmatic.execute import execute_command
+from borgmatic.execute import execute_command, execute_command_and_capture_output
 
 logger = logging.getLogger(__name__)
 
@@ -33,11 +33,8 @@ def resolve_archive_name(
         + flags.make_repository_flags(repository, local_borg_version)
     )
 
-    output = execute_command(
-        full_command,
-        output_log_level=None,
-        borg_local_path=local_path,
-        extra_environment=environment.make_environment(storage_config),
+    output = execute_command_and_capture_output(
+        full_command, extra_environment=environment.make_environment(storage_config),
     )
     try:
         latest_archive = output.strip().splitlines()[-1]
@@ -117,12 +114,12 @@ def list_repository(
         repository, storage_config, local_borg_version, rlist_arguments, local_path, remote_path
     )
 
-    output = execute_command(
-        main_command,
-        output_log_level=None if rlist_arguments.json else logging.WARNING,
-        borg_local_path=local_path,
-        extra_environment=borg_environment,
-    )
-
     if rlist_arguments.json:
-        return output
+        return execute_command_and_capture_output(main_command, extra_environment=borg_environment,)
+    else:
+        execute_command(
+            main_command,
+            output_log_level=logging.WARNING,
+            borg_local_path=local_path,
+            extra_environment=borg_environment,
+        )

borgmatic/borg/version.py

@@ -1,7 +1,7 @@
 import logging
 
 from borgmatic.borg import environment
-from borgmatic.execute import execute_command
+from borgmatic.execute import execute_command_and_capture_output
 
 logger = logging.getLogger(__name__)
 
@@ -18,11 +18,8 @@ def local_borg_version(storage_config, local_path='borg'):
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
     )
-    output = execute_command(
-        full_command,
-        output_log_level=None,
-        borg_local_path=local_path,
-        extra_environment=environment.make_environment(storage_config),
+    output = execute_command_and_capture_output(
+        full_command, extra_environment=environment.make_environment(storage_config),
     )
 
     try:

borgmatic/config/normalize.py

@@ -8,49 +8,53 @@ def normalize(config_filename, config):
     message warnings produced based on the normalization performed.
     '''
     logs = []
+    location = config.get('location') or {}
+    storage = config.get('storage') or {}
+    consistency = config.get('consistency') or {}
+    hooks = config.get('hooks') or {}
 
     # Upgrade exclude_if_present from a string to a list.
-    exclude_if_present = config.get('location', {}).get('exclude_if_present')
+    exclude_if_present = location.get('exclude_if_present')
     if isinstance(exclude_if_present, str):
         config['location']['exclude_if_present'] = [exclude_if_present]
 
     # Upgrade various monitoring hooks from a string to a dict.
-    healthchecks = config.get('hooks', {}).get('healthchecks')
+    healthchecks = hooks.get('healthchecks')
     if isinstance(healthchecks, str):
         config['hooks']['healthchecks'] = {'ping_url': healthchecks}
 
-    cronitor = config.get('hooks', {}).get('cronitor')
+    cronitor = hooks.get('cronitor')
     if isinstance(cronitor, str):
         config['hooks']['cronitor'] = {'ping_url': cronitor}
 
-    pagerduty = config.get('hooks', {}).get('pagerduty')
+    pagerduty = hooks.get('pagerduty')
     if isinstance(pagerduty, str):
         config['hooks']['pagerduty'] = {'integration_key': pagerduty}
 
-    cronhub = config.get('hooks', {}).get('cronhub')
+    cronhub = hooks.get('cronhub')
     if isinstance(cronhub, str):
         config['hooks']['cronhub'] = {'ping_url': cronhub}
 
     # Upgrade consistency checks from a list of strings to a list of dicts.
-    checks = config.get('consistency', {}).get('checks')
+    checks = consistency.get('checks')
     if isinstance(checks, list) and len(checks) and isinstance(checks[0], str):
         config['consistency']['checks'] = [{'name': check_type} for check_type in checks]
 
     # Rename various configuration options.
-    numeric_owner = config.get('location', {}).pop('numeric_owner', None)
+    numeric_owner = location.pop('numeric_owner', None)
     if numeric_owner is not None:
         config['location']['numeric_ids'] = numeric_owner
 
-    bsd_flags = config.get('location', {}).pop('bsd_flags', None)
+    bsd_flags = location.pop('bsd_flags', None)
     if bsd_flags is not None:
         config['location']['flags'] = bsd_flags
 
-    remote_rate_limit = config.get('storage', {}).pop('remote_rate_limit', None)
+    remote_rate_limit = storage.pop('remote_rate_limit', None)
     if remote_rate_limit is not None:
         config['storage']['upload_rate_limit'] = remote_rate_limit
 
     # Upgrade remote repositories to ssh:// syntax, required in Borg 2.
-    repositories = config.get('location', {}).get('repositories')
+    repositories = location.get('repositories')
     if repositories:
         config['location']['repositories'] = []
         for repository in repositories:

borgmatic/config/schema.yaml

@@ -764,6 +764,32 @@ properties:
                             description: |
                                 Path to a certificate revocation list.
                             example: "/root/.postgresql/root.crl"
+                        pg_dump_command:
+                            type: string
+                            description: |
+                                Command to use instead of "pg_dump" or
+                                "pg_dumpall". This can be used to run a specific
+                                pg_dump version (e.g., one inside a running
+                                docker container). Defaults to "pg_dump" for
+                                single database dump or "pg_dumpall" to dump
+                                all databases.
+                            example: docker exec my_pg_container pg_dump
+                        pg_restore_command:
+                            type: string
+                            description: |
+                                Command to use instead of "pg_restore". This
+                                can be used to run a specific pg_restore
+                                version (e.g., one inside a running docker
+                                container). Defaults to "pg_restore".
+                            example: docker exec my_pg_container pg_restore
+                        psql_command:
+                            type: string
+                            description: |
+                                Command to use instead of "psql". This can be
+                                used to run a specific psql version (e.g.,
+                                one inside a running docker container).
+                                Defaults to "psql".
+                            example: docker exec my_pg_container psql
                         options:
                             type: string
                             description: |

borgmatic/execute.py

@@ -147,7 +147,7 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
         }
 
 
-def log_command(full_command, input_file, output_file):
+def log_command(full_command, input_file=None, output_file=None):
     '''
     Log the given command (a sequence of command/argument strings), along with its input/output file
     paths.
@@ -178,15 +178,14 @@ def execute_command(
 ):
     '''
     Execute the given command (a sequence of command/argument strings) and log its output at the
-    given log level. If output log level is None, instead capture and return the output. (Implies
-    run_to_completion.) If an open output file object is given, then write stdout to the file and
-    only log stderr (but only if an output log level is set). If an open input file object is given,
-    then read stdin from the file. If shell is True, execute the command within a shell. If an extra
-    environment dict is given, then use it to augment the current environment, and pass the result
-    into the command. If a working directory is given, use that as the present working directory
-    when running the command. If a Borg local path is given, and the command matches it (regardless
-    of arguments), treat exit code 1 as a warning instead of an error. If run to completion is
-    False, then return the process for the command without executing it to completion.
+    given log level. If an open output file object is given, then write stdout to the file and only
+    log stderr. If an open input file object is given, then read stdin from the file. If shell is
+    True, execute the command within a shell. If an extra environment dict is given, then use it to
+    augment the current environment, and pass the result into the command. If a working directory is
+    given, use that as the present working directory when running the command. If a Borg local path
+    is given, and the command matches it (regardless of arguments), treat exit code 1 as a warning
+    instead of an error. If run to completion is False, then return the process for the command
+    without executing it to completion.
 
     Raise subprocesses.CalledProcessError if an error occurs while running the command.
     '''
@@ -195,12 +194,6 @@ def execute_command(
     do_not_capture = bool(output_file is DO_NOT_CAPTURE)
     command = ' '.join(full_command) if shell else full_command
 
-    if output_log_level is None:
-        output = subprocess.check_output(
-            command, stderr=subprocess.STDOUT, shell=shell, env=environment, cwd=working_directory
-        )
-        return output.decode() if output is not None else None
-
     process = subprocess.Popen(
         command,
         stdin=input_file,
@@ -218,6 +211,33 @@ def execute_command(
     )
 
 
+def execute_command_and_capture_output(
+    full_command, capture_stderr=False, shell=False, extra_environment=None, working_directory=None,
+):
+    '''
+    Execute the given command (a sequence of command/argument strings), capturing and returning its
+    output (stdout). If capture stderr is True, then capture and return stderr in addition to
+    stdout. If shell is True, execute the command within a shell. If an extra environment dict is
+    given, then use it to augment the current environment, and pass the result into the command. If
+    a working directory is given, use that as the present working directory when running the command.
+
+    Raise subprocesses.CalledProcessError if an error occurs while running the command.
+    '''
+    log_command(full_command)
+    environment = {**os.environ, **extra_environment} if extra_environment else None
+    command = ' '.join(full_command) if shell else full_command
+
+    output = subprocess.check_output(
+        command,
+        stderr=subprocess.STDOUT if capture_stderr else None,
+        shell=shell,
+        env=environment,
+        cwd=working_directory,
+    )
+
+    return output.decode() if output is not None else None
+
+
 def execute_command_with_processes(
     full_command,
     processes,

borgmatic/hooks/mysql.py

@@ -1,6 +1,10 @@
 import logging
 
-from borgmatic.execute import execute_command, execute_command_with_processes
+from borgmatic.execute import (
+    execute_command,
+    execute_command_and_capture_output,
+    execute_command_with_processes,
+)
 from borgmatic.hooks import dump
 
 logger = logging.getLogger(__name__)
@@ -42,8 +46,8 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run_labe
     logger.debug(
         '{}: Querying for "all" MySQL databases to dump{}'.format(log_prefix, dry_run_label)
     )
-    show_output = execute_command(
-        show_command, output_log_level=None, extra_environment=extra_environment
+    show_output = execute_command_and_capture_output(
+        show_command, extra_environment=extra_environment
     )
 
     return tuple(

borgmatic/hooks/postgresql.py

@@ -56,13 +56,10 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
         )
         all_databases = bool(name == 'all')
         dump_format = database.get('format', 'custom')
+        default_dump_command = 'pg_dumpall' if all_databases else 'pg_dump'
+        dump_command = database.get('pg_dump_command') or default_dump_command
         command = (
-            (
-                'pg_dumpall' if all_databases else 'pg_dump',
-                '--no-password',
-                '--clean',
-                '--if-exists',
-            )
+            (dump_command, '--no-password', '--clean', '--if-exists',)
             + (('--host', database['hostname']) if 'hostname' in database else ())
             + (('--port', str(database['port'])) if 'port' in database else ())
             + (('--username', database['username']) if 'username' in database else ())
@@ -140,16 +137,18 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
     dump_filename = dump.make_database_dump_filename(
         make_dump_path(location_config), database['name'], database.get('hostname')
     )
+    psql_command = database.get('psql_command') or 'psql'
     analyze_command = (
-        ('psql', '--no-password', '--quiet')
+        (psql_command, '--no-password', '--quiet')
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--username', database['username']) if 'username' in database else ())
         + (('--dbname', database['name']) if not all_databases else ())
         + ('--command', 'ANALYZE')
     )
+    pg_restore_command = database.get('pg_restore_command') or 'pg_restore'
     restore_command = (
-        ('psql' if all_databases else 'pg_restore', '--no-password')
+        (psql_command if all_databases else pg_restore_command, '--no-password')
         + (
             ('--if-exists', '--exit-on-error', '--clean', '--dbname', database['name'])
             if not all_databases

docs/how-to/backup-your-databases.md

@@ -220,8 +220,8 @@ such files. Common directories to exclude are `/dev` and `/run`, but that may
 not be exhaustive. <span class="minilink minilink-addedin">New in version
 1.7.3</span> When database hooks are enabled, borgmatic automatically excludes
 special files that may cause Borg to hang, so you no longer need to manually
-exclude them. You can override/prevent this behavior by explicitly setting
-`read_special` to true.
+exclude them. (This includes symlinks with special files as a destination.) You
+can override/prevent this behavior by explicitly setting `read_special` to true.
 
 
 ### Manual restoration

docs/how-to/deal-with-very-large-backups.md

@@ -44,9 +44,11 @@ consistency checks with `check` on a much less frequent basis (e.g. with
 
 ### Consistency check configuration
 
-Another option is to customize your consistency checks. The default
-consistency checks run both full-repository checks and per-archive checks
-within each repository no more than once a month.
+Another option is to customize your consistency checks. By default, if you
+omit consistency checks from configuration, borgmatic runs full-repository
+checks (`repository`) and per-archive checks (`archives`) within each
+repository, no more than once a month. This is equivalent to what `borg check`
+does if run without options.
 
 But if you find that archive checks are too slow, for example, you can
 configure borgmatic to run repository checks only. Configure this in the

docs/how-to/make-backups-redundant.md

@@ -42,3 +42,13 @@ potentially across providers.
 See [Borg repository URLs
 documentation](https://borgbackup.readthedocs.io/en/stable/usage/general.html#repository-urls)
 for more information on how to specify local and remote repository paths.
+
+### Different options per repository
+
+What if you want borgmatic to backup to multiple repositories—while also
+setting different options for each one? In that case, you'll need to use
+[a separate borgmatic configuration file for each
+repository](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/)
+instead of the multiple repositories in one configuration file as described
+above. That's because all of the repositories in a particular configuration
+file get the same options applied.

docs/how-to/make-per-application-backups.md

@@ -106,11 +106,60 @@ But if you do want to merge in a YAML key *and* its values, keep reading!
 
 ## Include merging
 
-If you need to get even fancier and pull in common configuration options while
-potentially overriding individual options, you can perform a YAML merge of
-included configuration using the YAML `<<` key. For instance, here's an
-example of a main configuration file that pulls in two retention options via
-an include and then overrides one of them locally:
+If you need to get even fancier and merge in common configuration options, you
+can perform a YAML merge of included configuration using the YAML `<<` key.
+For instance, here's an example of a main configuration file that pulls in
+retention and consistency options via a single include:
+
+```yaml
+<<: !include /etc/borgmatic/common.yaml
+
+location:
+   ...
+```
+
+This is what `common.yaml` might look like:
+
+```yaml
+retention:
+    keep_hourly: 24
+    keep_daily: 7
+
+consistency:
+    checks:
+        - name: repository
+```
+
+Once this include gets merged in, the resulting configuration would have all
+of the `location` options from the original configuration file *and* the
+`retention` and `consistency` options from the include.
+
+Prior to borgmatic version 1.6.0, when there's a section collision between the
+local file and the merged include, the local file's section takes precedence.
+So if the `retention` section appears in both the local file and the include
+file, the included `retention` is ignored in favor of the local `retention`.
+But see below about deep merge in version 1.6.0+.
+
+Note that this `<<` include merging syntax is only for merging in mappings
+(configuration options and their values). But if you'd like to include a
+single value directly, please see the section above about standard includes.
+
+Additionally, there is a limitation preventing multiple `<<` include merges
+per section. So for instance, that means you can do one `<<` merge at the
+global level, another `<<` within each configuration section, etc. (This is a
+YAML limitation.)
+
+
+### Deep merge
+
+<span class="minilink minilink-addedin">New in version 1.6.0</span> borgmatic
+performs a deep merge of merged include files, meaning that values are merged
+at all levels in the two configuration files. This allows you to include
+common configuration—up to full borgmatic configuration files—while overriding
+only the parts you want to customize.
+
+For instance, here's an example of a main configuration file that pulls in two
+retention options via an include and then overrides one of them locally:
 
 ```yaml
 <<: !include /etc/borgmatic/common.yaml
@@ -136,24 +185,8 @@ Once this include gets merged in, the resulting configuration would have a
 When there's an option collision between the local file and the merged
 include, the local file's option takes precedence.
 
-Note that this `<<` include merging syntax is only for merging in mappings
-(configuration options and their values). But if you'd like to include a
-single value directly, please see the section above about standard includes.
-
-Additionally, there is a limitation preventing multiple `<<` include merges
-per section. So for instance, that means you can do one `<<` merge at the
-global level, another `<<` within each configuration section, etc. (This is a
-YAML limitation.)
-
-
-### Deep merge
-
-<span class="minilink minilink-addedin">New in version 1.6.0</span> borgmatic
-performs a deep merge of merged include files, meaning that values are merged
-at all levels in the two configuration files. Colliding list values are
-appended together. This allows you to include common configuration—up to full
-borgmatic configuration files—while overriding only the parts you want to
-customize.
+<span class="minilink minilink-addedin">New in version 1.6.1</span> Colliding
+list values are appended together.
 
 
 ## Configuration overrides

docs/how-to/upgrade.md

@@ -160,17 +160,22 @@ Then, run the `rcreate` action (formerly `init`) to create that new Borg 2
 repository:
 
 ```bash
-borgmatic rcreate --verbosity 1 --encryption repokey-aes-ocb \
+borgmatic rcreate --verbosity 1 --encryption repokey-blake2-aes-ocb \
     --source-repository original.borg --repository upgraded.borg
 ```
 
-(Note that `repokey-chacha20-poly1305` may be faster than `repokey-aes-ocb` on
-certain platforms like ARM64.)
-
 This creates an empty repository and doesn't actually transfer any data yet.
 The `--source-repository` flag is necessary to reuse key material from your
 Borg 1 repository so that the subsequent data transfer can work.
 
+The `--encryption` value above selects the same chunk ID algorithm (`blake2`)
+used in Borg 1, thereby making deduplication work across transferred archives
+and new archives. Note that `repokey-blake2-chacha20-poly1305` may be faster
+than `repokey-blake2-aes-ocb` on certain platforms like ARM64. Read about
+[Borg encryption
+modes](https://borgbackup.readthedocs.io/en/2.0.0b3/usage/rcreate.html#encryption-mode-tldr)
+for the menu of available encryption modes.
+
 To transfer data from your original Borg 1 repository to your newly created
 Borg 2 repository:
 
@@ -191,7 +196,7 @@ confirmation of success—or tells you if something hasn't been transferred yet.
 Note that by omitting the `--upgrader` flag, you can also do archive transfers
 between Borg 2 repositories without upgrading, even down to individual
 archives. For more on that functionality, see the [Borg transfer
-documentation](https://borgbackup.readthedocs.io/en/2.0.0b1/usage/transfer.html).
+documentation](https://borgbackup.readthedocs.io/en/2.0.0b3/usage/transfer.html).
 
 That's it! Now you can use your new Borg 2 repository as normal with
 borgmatic. If you've got multiple repositories, repeat the above process for

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.7.3'
+VERSION = '1.7.5'
 
 
 setup(

test_requirements.txt

@@ -14,8 +14,8 @@ py==1.10.0
 pycodestyle==2.8.0
 pyflakes==2.4.0
 jsonschema==3.2.0
-pytest==6.2.5
-pytest-cov==3.0.0
+pytest==7.2.0
+pytest-cov==4.0.0
 regex; python_version >= '3.8'
 requests==2.25.0
 ruamel.yaml>0.15.0,<0.18.0

tests/unit/borg/test_check.py

@@ -265,6 +265,14 @@ def test_make_check_flags_with_archives_check_and_last_includes_last_flag():
     assert flags == ('--archives-only', '--last', '3')
 
 
+def test_make_check_flags_with_data_check_and_last_includes_last_flag():
+    flexmock(module.feature).should_receive('available').and_return(True)
+
+    flags = module.make_check_flags('1.2.3', ('data',), check_last=3)
+
+    assert flags == ('--archives-only', '--last', '3', '--verify-data')
+
+
 def test_make_check_flags_with_repository_check_and_last_omits_last_flag():
     flexmock(module.feature).should_receive('available').and_return(True)
 
@@ -289,6 +297,14 @@ def test_make_check_flags_with_archives_check_and_prefix_includes_match_archives
     assert flags == ('--archives-only', '--match-archives', 'sh:foo-*')
 
 
+def test_make_check_flags_with_data_check_and_prefix_includes_match_archives_flag():
+    flexmock(module.feature).should_receive('available').and_return(True)
+
+    flags = module.make_check_flags('1.2.3', ('data',), prefix='foo-')
+
+    assert flags == ('--archives-only', '--match-archives', 'sh:foo-*', '--verify-data')
+
+
 def test_make_check_flags_with_archives_check_and_empty_prefix_omits_match_archives_flag():
     flexmock(module.feature).should_receive('available').and_return(True)
 

tests/unit/borg/test_create.py

@@ -338,6 +338,12 @@ def test_special_file_looks_at_file_type(character_device, block_device, fifo, e
     assert module.special_file('/dev/special') == expected_result
 
 
+def test_special_file_treats_broken_symlink_as_non_special():
+    flexmock(module.os).should_receive('stat').and_raise(FileNotFoundError)
+
+    assert module.special_file('/broken/symlink') is False
+
+
 def test_any_parent_directories_treats_parents_as_match():
     module.any_parent_directories('/foo/bar.txt', ('/foo', '/etc'))
 
@@ -351,7 +357,7 @@ def test_any_parent_directories_treats_unrelated_paths_as_non_match():
 
 
 def test_collect_special_file_paths_parses_special_files_from_borg_dry_run_file_list():
-    flexmock(module).should_receive('execute_command').and_return(
+    flexmock(module).should_receive('execute_command_and_capture_output').and_return(
         'Processing files ...\n- /foo\n- /bar\n- /baz'
     )
     flexmock(module).should_receive('special_file').and_return(True)
@@ -367,7 +373,9 @@ def test_collect_special_file_paths_parses_special_files_from_borg_dry_run_file_
 
 
 def test_collect_special_file_paths_excludes_requested_directories():
-    flexmock(module).should_receive('execute_command').and_return('- /foo\n- /bar\n- /baz')
+    flexmock(module).should_receive('execute_command_and_capture_output').and_return(
+        '- /foo\n- /bar\n- /baz'
+    )
     flexmock(module).should_receive('special_file').and_return(True)
     flexmock(module).should_receive('any_parent_directories').and_return(False).and_return(
         True
@@ -383,7 +391,9 @@ def test_collect_special_file_paths_excludes_requested_directories():
 
 
 def test_collect_special_file_paths_excludes_non_special_files():
-    flexmock(module).should_receive('execute_command').and_return('- /foo\n- /bar\n- /baz')
+    flexmock(module).should_receive('execute_command_and_capture_output').and_return(
+        '- /foo\n- /bar\n- /baz'
+    )
     flexmock(module).should_receive('special_file').and_return(True).and_return(False).and_return(
         True
     )
@@ -622,11 +632,8 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
         (f'repo::{DEFAULT_ARCHIVE_NAME}',)
     )
     flexmock(module.environment).should_receive('make_environment')
-    flexmock(module).should_receive('execute_command').with_args(
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('--json',),
-        output_log_level=None,
-        output_file=None,
-        borg_local_path='borg',
         working_directory=None,
         extra_environment=None,
     )
@@ -703,11 +710,8 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
         (f'repo::{DEFAULT_ARCHIVE_NAME}',)
     )
     flexmock(module.environment).should_receive('make_environment')
-    flexmock(module).should_receive('execute_command').with_args(
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('--json',),
-        output_log_level=None,
-        output_file=None,
-        borg_local_path='borg',
         working_directory=None,
         extra_environment=None,
     )
@@ -1997,11 +2001,8 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
         (f'repo::{DEFAULT_ARCHIVE_NAME}',)
     )
     flexmock(module.environment).should_receive('make_environment')
-    flexmock(module).should_receive('execute_command').with_args(
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('--json',),
-        output_log_level=None,
-        output_file=None,
-        borg_local_path='borg',
         working_directory=None,
         extra_environment=None,
     ).and_return('[]')
@@ -2039,11 +2040,8 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter()
         (f'repo::{DEFAULT_ARCHIVE_NAME}',)
     )
     flexmock(module.environment).should_receive('make_environment')
-    flexmock(module).should_receive('execute_command').with_args(
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'create') + REPO_ARCHIVE_WITH_PATHS + ('--json',),
-        output_log_level=None,
-        output_file=None,
-        borg_local_path='borg',
         working_directory=None,
         extra_environment=None,
     ).and_return('[]')

tests/unit/borg/test_extract.py

@@ -15,13 +15,6 @@ def insert_execute_command_mock(command, working_directory=None):
     ).once()
 
 
-def insert_execute_command_output_mock(command, result):
-    flexmock(module.environment).should_receive('make_environment')
-    flexmock(module).should_receive('execute_command').with_args(
-        command, output_log_level=None, borg_local_path=command[0], extra_environment=None,
-    ).and_return(result).once()
-
-
 def test_extract_last_archive_dry_run_calls_borg_with_last_archive():
     flexmock(module.rlist).should_receive('resolve_archive_name').and_return('archive')
     insert_execute_command_mock(('borg', 'extract', '--dry-run', 'repo::archive'))

tests/unit/borg/test_info.py

@@ -53,11 +53,8 @@ def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_outpu
     flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',))
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo'))
     flexmock(module.environment).should_receive('make_environment')
-    flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', '--json', '--repo', 'repo'),
-        output_log_level=None,
-        borg_local_path='borg',
-        extra_environment=None,
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
+        ('borg', 'info', '--json', '--repo', 'repo'), extra_environment=None,
     ).and_return('[]')
 
     insert_logging_mock(logging.INFO)
@@ -97,11 +94,8 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp
     flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',))
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo'))
     flexmock(module.environment).should_receive('make_environment')
-    flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', '--json', '--repo', 'repo'),
-        output_log_level=None,
-        borg_local_path='borg',
-        extra_environment=None,
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
+        ('borg', 'info', '--json', '--repo', 'repo'), extra_environment=None,
     ).and_return('[]')
 
     insert_logging_mock(logging.DEBUG)
@@ -120,11 +114,8 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter():
     flexmock(module.flags).should_receive('make_flags_from_arguments').and_return(('--json',))
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo'))
     flexmock(module.environment).should_receive('make_environment')
-    flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'info', '--json', '--repo', 'repo'),
-        output_log_level=None,
-        borg_local_path='borg',
-        extra_environment=None,
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
+        ('borg', 'info', '--json', '--repo', 'repo'), extra_environment=None,
     ).and_return('[]')
 
     json_output = module.display_archives_info(

tests/unit/borg/test_list.py

@@ -361,11 +361,8 @@ def test_list_archive_calls_borg_multiple_times_with_find_paths():
 
     flexmock(module.feature).should_receive('available').and_return(False)
     flexmock(module.rlist).should_receive('make_rlist_command').and_return(('borg', 'list', 'repo'))
-    flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list', 'repo'),
-        output_log_level=None,
-        borg_local_path='borg',
-        extra_environment=None,
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
+        ('borg', 'list', 'repo'), extra_environment=None,
     ).and_return('archive1\narchive2').once()
     flexmock(module).should_receive('make_list_command').and_return(
         ('borg', 'list', 'repo::archive1')
@@ -559,11 +556,8 @@ def test_list_archive_with_find_paths_allows_archive_filter_flag_but_only_passes
         remote_path=None,
     ).and_return(('borg', 'rlist', '--repo', 'repo'))
 
-    flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'rlist', '--repo', 'repo'),
-        output_log_level=None,
-        borg_local_path='borg',
-        extra_environment=None,
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
+        ('borg', 'rlist', '--repo', 'repo'), extra_environment=None,
     ).and_return('archive1\narchive2').once()
 
     flexmock(module).should_receive('make_list_command').with_args(

tests/unit/borg/test_rinfo.py

@@ -68,11 +68,8 @@ def test_display_repository_info_with_log_info_and_json_suppresses_most_borg_out
     flexmock(module.feature).should_receive('available').and_return(True)
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',))
     flexmock(module.environment).should_receive('make_environment')
-    flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'rinfo', '--json', '--repo', 'repo'),
-        output_log_level=None,
-        borg_local_path='borg',
-        extra_environment=None,
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
+        ('borg', 'rinfo', '--json', '--repo', 'repo'), extra_environment=None,
     ).and_return('[]')
 
     insert_logging_mock(logging.INFO)
@@ -110,11 +107,8 @@ def test_display_repository_info_with_log_debug_and_json_suppresses_most_borg_ou
     flexmock(module.feature).should_receive('available').and_return(True)
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',))
     flexmock(module.environment).should_receive('make_environment')
-    flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'rinfo', '--json', '--repo', 'repo'),
-        output_log_level=None,
-        borg_local_path='borg',
-        extra_environment=None,
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
+        ('borg', 'rinfo', '--json', '--repo', 'repo'), extra_environment=None,
     ).and_return('[]')
 
     insert_logging_mock(logging.DEBUG)
@@ -132,11 +126,8 @@ def test_display_repository_info_with_json_calls_borg_with_json_parameter():
     flexmock(module.feature).should_receive('available').and_return(True)
     flexmock(module.flags).should_receive('make_repository_flags').and_return(('--repo', 'repo',))
     flexmock(module.environment).should_receive('make_environment')
-    flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'rinfo', '--json', '--repo', 'repo'),
-        output_log_level=None,
-        borg_local_path='borg',
-        extra_environment=None,
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
+        ('borg', 'rinfo', '--json', '--repo', 'repo'), extra_environment=None,
     ).and_return('[]')
 
     json_output = module.display_repository_info(

tests/unit/borg/test_rlist.py

@@ -28,11 +28,8 @@ def test_resolve_archive_name_passes_through_non_latest_archive_name():
 def test_resolve_archive_name_calls_borg_with_parameters():
     expected_archive = 'archive-name'
     flexmock(module.environment).should_receive('make_environment')
-    flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
-        output_log_level=None,
-        borg_local_path='borg',
-        extra_environment=None,
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
+        ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None,
     ).and_return(expected_archive + '\n')
 
     assert (
@@ -44,11 +41,8 @@ def test_resolve_archive_name_calls_borg_with_parameters():
 def test_resolve_archive_name_with_log_info_calls_borg_without_info_parameter():
     expected_archive = 'archive-name'
     flexmock(module.environment).should_receive('make_environment')
-    flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
-        output_log_level=None,
-        borg_local_path='borg',
-        extra_environment=None,
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
+        ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None,
     ).and_return(expected_archive + '\n')
     insert_logging_mock(logging.INFO)
 
@@ -61,11 +55,8 @@ def test_resolve_archive_name_with_log_info_calls_borg_without_info_parameter():
 def test_resolve_archive_name_with_log_debug_calls_borg_without_debug_parameter():
     expected_archive = 'archive-name'
     flexmock(module.environment).should_receive('make_environment')
-    flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
-        output_log_level=None,
-        borg_local_path='borg',
-        extra_environment=None,
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
+        ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None,
     ).and_return(expected_archive + '\n')
     insert_logging_mock(logging.DEBUG)
 
@@ -78,11 +69,8 @@ def test_resolve_archive_name_with_log_debug_calls_borg_without_debug_parameter(
 def test_resolve_archive_name_with_local_path_calls_borg_via_local_path():
     expected_archive = 'archive-name'
     flexmock(module.environment).should_receive('make_environment')
-    flexmock(module).should_receive('execute_command').with_args(
-        ('borg1', 'list') + BORG_LIST_LATEST_ARGUMENTS,
-        output_log_level=None,
-        borg_local_path='borg1',
-        extra_environment=None,
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
+        ('borg1', 'list') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None,
     ).and_return(expected_archive + '\n')
 
     assert (
@@ -96,10 +84,8 @@ def test_resolve_archive_name_with_local_path_calls_borg_via_local_path():
 def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_parameters():
     expected_archive = 'archive-name'
     flexmock(module.environment).should_receive('make_environment')
-    flexmock(module).should_receive('execute_command').with_args(
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'list', '--remote-path', 'borg1') + BORG_LIST_LATEST_ARGUMENTS,
-        output_log_level=None,
-        borg_local_path='borg',
         extra_environment=None,
     ).and_return(expected_archive + '\n')
 
@@ -113,11 +99,8 @@ def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_param
 
 def test_resolve_archive_name_without_archives_raises():
     flexmock(module.environment).should_receive('make_environment')
-    flexmock(module).should_receive('execute_command').with_args(
-        ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
-        output_log_level=None,
-        borg_local_path='borg',
-        extra_environment=None,
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
+        ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS, extra_environment=None,
     ).and_return('')
 
     with pytest.raises(ValueError):
@@ -128,10 +111,8 @@ def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_parameter
     expected_archive = 'archive-name'
 
     flexmock(module.environment).should_receive('make_environment')
-    flexmock(module).should_receive('execute_command').with_args(
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('borg', 'list', '--lock-wait', 'okay') + BORG_LIST_LATEST_ARGUMENTS,
-        output_log_level=None,
-        borg_local_path='borg',
         extra_environment=None,
     ).and_return(expected_archive + '\n')
 
@@ -385,7 +366,7 @@ def test_list_repository_with_json_returns_borg_output():
         remote_path=None,
     ).and_return(('borg', 'rlist', 'repo'))
     flexmock(module.environment).should_receive('make_environment')
-    flexmock(module).should_receive('execute_command').and_return(json_output)
+    flexmock(module).should_receive('execute_command_and_capture_output').and_return(json_output)
 
     assert (
         module.list_repository(

tests/unit/borg/test_version.py

@@ -10,22 +10,24 @@ from ..test_verbosity import insert_logging_mock
 VERSION = '1.2.3'
 
 
-def insert_execute_command_mock(command, borg_local_path='borg', version_output=f'borg {VERSION}'):
+def insert_execute_command_and_capture_output_mock(
+    command, borg_local_path='borg', version_output=f'borg {VERSION}'
+):
     flexmock(module.environment).should_receive('make_environment')
-    flexmock(module).should_receive('execute_command').with_args(
-        command, output_log_level=None, borg_local_path=borg_local_path, extra_environment=None,
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
+        command, extra_environment=None,
     ).once().and_return(version_output)
 
 
 def test_local_borg_version_calls_borg_with_required_parameters():
-    insert_execute_command_mock(('borg', '--version'))
+    insert_execute_command_and_capture_output_mock(('borg', '--version'))
     flexmock(module.environment).should_receive('make_environment')
 
     assert module.local_borg_version({}) == VERSION
 
 
 def test_local_borg_version_with_log_info_calls_borg_with_info_parameter():
-    insert_execute_command_mock(('borg', '--version', '--info'))
+    insert_execute_command_and_capture_output_mock(('borg', '--version', '--info'))
     insert_logging_mock(logging.INFO)
     flexmock(module.environment).should_receive('make_environment')
 
@@ -33,7 +35,7 @@ def test_local_borg_version_with_log_info_calls_borg_with_info_parameter():
 
 
 def test_local_borg_version_with_log_debug_calls_borg_with_debug_parameters():
-    insert_execute_command_mock(('borg', '--version', '--debug', '--show-rc'))
+    insert_execute_command_and_capture_output_mock(('borg', '--version', '--debug', '--show-rc'))
     insert_logging_mock(logging.DEBUG)
     flexmock(module.environment).should_receive('make_environment')
 
@@ -41,14 +43,14 @@ def test_local_borg_version_with_log_debug_calls_borg_with_debug_parameters():
 
 
 def test_local_borg_version_with_local_borg_path_calls_borg_with_it():
-    insert_execute_command_mock(('borg1', '--version'), borg_local_path='borg1')
+    insert_execute_command_and_capture_output_mock(('borg1', '--version'), borg_local_path='borg1')
     flexmock(module.environment).should_receive('make_environment')
 
     assert module.local_borg_version({}, 'borg1') == VERSION
 
 
 def test_local_borg_version_with_invalid_version_raises():
-    insert_execute_command_mock(('borg', '--version'), version_output='wtf')
+    insert_execute_command_and_capture_output_mock(('borg', '--version'), version_output='wtf')
     flexmock(module.environment).should_receive('make_environment')
 
     with pytest.raises(ValueError):

tests/unit/config/test_normalize.py

@@ -21,11 +21,13 @@ from borgmatic.config import normalize as module
             {'location': {'source_directories': ['foo', 'bar']}},
             False,
         ),
+        ({'location': None}, {'location': None}, False,),
         (
             {'storage': {'compression': 'yes_please'}},
             {'storage': {'compression': 'yes_please'}},
             False,
         ),
+        ({'storage': None}, {'storage': None}, False,),
         (
             {'hooks': {'healthchecks': 'https://example.com'}},
             {'hooks': {'healthchecks': {'ping_url': 'https://example.com'}}},
@@ -46,11 +48,18 @@ from borgmatic.config import normalize as module
             {'hooks': {'cronhub': {'ping_url': 'https://example.com'}}},
             False,
         ),
+        ({'hooks': None}, {'hooks': None}, False,),
         (
             {'consistency': {'checks': ['archives']}},
             {'consistency': {'checks': [{'name': 'archives'}]}},
             False,
         ),
+        (
+            {'consistency': {'checks': ['archives']}},
+            {'consistency': {'checks': [{'name': 'archives'}]}},
+            False,
+        ),
+        ({'consistency': None}, {'consistency': None}, False,),
         ({'location': {'numeric_owner': False}}, {'location': {'numeric_ids': False}}, False,),
         ({'location': {'bsd_flags': False}}, {'location': {'flags': False}}, False,),
         (

tests/unit/hooks/test_mysql.py

@@ -22,9 +22,8 @@ def test_database_names_to_dump_queries_mysql_for_database_names():
     extra_environment = flexmock()
     log_prefix = ''
     dry_run_label = ''
-    flexmock(module).should_receive('execute_command').with_args(
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         ('mysql', '--skip-column-names', '--batch', '--execute', 'show schemas'),
-        output_log_level=None,
         extra_environment=extra_environment,
     ).and_return('foo\nbar\nmysql\n').once()
 
@@ -200,7 +199,7 @@ def test_dump_databases_runs_mysqldump_for_all_databases():
 
 def test_database_names_to_dump_runs_mysql_with_list_options():
     database = {'name': 'all', 'list_options': '--defaults-extra-file=my.cnf'}
-    flexmock(module).should_receive('execute_command').with_args(
+    flexmock(module).should_receive('execute_command_and_capture_output').with_args(
         (
             'mysql',
             '--defaults-extra-file=my.cnf',
@@ -209,7 +208,6 @@ def test_database_names_to_dump_runs_mysql_with_list_options():
             '--execute',
             'show schemas',
         ),
-        output_log_level=None,
         extra_environment=None,
     ).and_return(('foo\nbar')).once()
 

tests/unit/hooks/test_postgresql.py

@@ -223,6 +223,36 @@ def test_dump_databases_runs_pg_dumpall_for_all_databases():
     assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
 
 
+def test_dump_databases_runs_non_default_pg_dump():
+    databases = [{'name': 'foo', 'pg_dump_command': 'special_pg_dump'}]
+    process = flexmock()
+    flexmock(module).should_receive('make_dump_path').and_return('')
+    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
+        'databases/localhost/foo'
+    )
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+    flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
+
+    flexmock(module).should_receive('execute_command').with_args(
+        (
+            'special_pg_dump',
+            '--no-password',
+            '--clean',
+            '--if-exists',
+            '--format',
+            'custom',
+            'foo',
+            '>',
+            'databases/localhost/foo',
+        ),
+        shell=True,
+        extra_environment={'PGSSLMODE': 'disable'},
+        run_to_completion=False,
+    ).and_return(process).once()
+
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
+
+
 def test_restore_database_dump_runs_pg_restore():
     database_config = [{'name': 'foo'}]
     extract_process = flexmock(stdout=flexmock())
@@ -388,6 +418,40 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
     )
 
 
+def test_restore_database_dump_runs_non_default_pg_restore_and_psql():
+    database_config = [
+        {'name': 'foo', 'pg_restore_command': 'special_pg_restore', 'psql_command': 'special_psql'}
+    ]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
+    flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        (
+            'special_pg_restore',
+            '--no-password',
+            '--if-exists',
+            '--exit-on-error',
+            '--clean',
+            '--dbname',
+            'foo',
+        ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment={'PGSSLMODE': 'disable'},
+    ).once()
+    flexmock(module).should_receive('execute_command').with_args(
+        ('special_psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'),
+        extra_environment={'PGSSLMODE': 'disable'},
+    ).once()
+
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+    )
+
+
 def test_restore_database_dump_with_dry_run_skips_restore():
     database_config = [{'name': 'foo'}]
 

tests/unit/test_execute.py

@@ -213,7 +213,20 @@ def test_execute_command_without_run_to_completion_returns_process():
     assert module.execute_command(full_command, run_to_completion=False) == process
 
 
-def test_execute_command_captures_output():
+def test_execute_command_and_capture_output_returns_stdout():
+    full_command = ['foo', 'bar']
+    expected_output = '[]'
+    flexmock(module.os, environ={'a': 'b'})
+    flexmock(module.subprocess).should_receive('check_output').with_args(
+        full_command, stderr=None, shell=False, env=None, cwd=None
+    ).and_return(flexmock(decode=lambda: expected_output)).once()
+
+    output = module.execute_command_and_capture_output(full_command)
+
+    assert output == expected_output
+
+
+def test_execute_command_and_capture_output_with_capture_stderr_returns_stderr():
     full_command = ['foo', 'bar']
     expected_output = '[]'
     flexmock(module.os, environ={'a': 'b'})
@@ -221,53 +234,49 @@ def test_execute_command_captures_output():
         full_command, stderr=module.subprocess.STDOUT, shell=False, env=None, cwd=None
     ).and_return(flexmock(decode=lambda: expected_output)).once()
 
-    output = module.execute_command(full_command, output_log_level=None)
+    output = module.execute_command_and_capture_output(full_command, capture_stderr=True)
 
     assert output == expected_output
 
 
-def test_execute_command_captures_output_with_shell():
+def test_execute_command_and_capture_output_returns_output_with_shell():
     full_command = ['foo', 'bar']
     expected_output = '[]'
     flexmock(module.os, environ={'a': 'b'})
     flexmock(module.subprocess).should_receive('check_output').with_args(
-        'foo bar', stderr=module.subprocess.STDOUT, shell=True, env=None, cwd=None
+        'foo bar', stderr=None, shell=True, env=None, cwd=None
     ).and_return(flexmock(decode=lambda: expected_output)).once()
 
-    output = module.execute_command(full_command, output_log_level=None, shell=True)
+    output = module.execute_command_and_capture_output(full_command, shell=True)
 
     assert output == expected_output
 
 
-def test_execute_command_captures_output_with_extra_environment():
+def test_execute_command_and_capture_output_returns_output_with_extra_environment():
     full_command = ['foo', 'bar']
     expected_output = '[]'
     flexmock(module.os, environ={'a': 'b'})
     flexmock(module.subprocess).should_receive('check_output').with_args(
-        full_command,
-        stderr=module.subprocess.STDOUT,
-        shell=False,
-        env={'a': 'b', 'c': 'd'},
-        cwd=None,
+        full_command, stderr=None, shell=False, env={'a': 'b', 'c': 'd'}, cwd=None,
     ).and_return(flexmock(decode=lambda: expected_output)).once()
 
-    output = module.execute_command(
-        full_command, output_log_level=None, shell=False, extra_environment={'c': 'd'}
+    output = module.execute_command_and_capture_output(
+        full_command, shell=False, extra_environment={'c': 'd'}
     )
 
     assert output == expected_output
 
 
-def test_execute_command_captures_output_with_working_directory():
+def test_execute_command_and_capture_output_returns_output_with_working_directory():
     full_command = ['foo', 'bar']
     expected_output = '[]'
     flexmock(module.os, environ={'a': 'b'})
     flexmock(module.subprocess).should_receive('check_output').with_args(
-        full_command, stderr=module.subprocess.STDOUT, shell=False, env=None, cwd='/working'
+        full_command, stderr=None, shell=False, env=None, cwd='/working'
     ).and_return(flexmock(decode=lambda: expected_output)).once()
 
-    output = module.execute_command(
-        full_command, output_log_level=None, shell=False, working_directory='/working'
+    output = module.execute_command_and_capture_output(
+        full_command, shell=False, working_directory='/working'
     )
 
     assert output == expected_output