Fix duplicate logging to Healthchecks and send "after_*" hooks output to Healthchecks (#328).

Reviewed-on: https://projects.torsion.org/witten/borgmatic/pulls/327

NEWS

@@ -1,3 +1,15 @@
+1.5.7.dev0
+ * #327: Fix broken pass-through of BORG_* environment variables to Borg.
+ * #328: Fix duplicate logging to Healthchecks and send "after_*" hooks output to Healthchecks.
+ * #331: Add SSL support to PostgreSQL database configuration.
+ * #333: Fix for potential data loss (data not getting backed up) when borgmatic omitted configured
+   source directories in certain situations. Specifically, this occurred when two source directories
+   on different filesystems were related by parentage (e.g. "/foo" and "/foo/bar/baz") and the
+   one_file_system option was enabled.
+ * Update documentation code fragments theme to better match the rest of the page.
+ * Improve configuration reference documentation readability via more aggressive word-wrapping in
+   configuration schema descriptions.
+
 1.5.6
  * #292: Allow before_backup and similiar hooks to exit with a soft failure without altering the
    monitoring status on Healthchecks or other providers. Support this by waiting to ping monitoring

borgmatic/borg/create.py

@@ -44,10 +44,24 @@ def _expand_home_directories(directories):
     return tuple(os.path.expanduser(directory) for directory in directories)
 
 
-def deduplicate_directories(directories):
+def map_directories_to_devices(directories):  # pragma: no cover
     '''
-    Given a sequence of directories, return them as a sorted tuple with all duplicate child
+    Given a sequence of directories, return a map from directory to an identifier for the device on
-    directories removed. For instance, if paths is ('/foo', '/foo/bar'), return just: ('/foo',)
+    which that directory resides. This is handy for determining whether two different directories
+    are on the same filesystem (have the same device identifier).
+    '''
+    return {directory: os.stat(directory).st_dev for directory in directories}
+
+
+def deduplicate_directories(directory_devices):
+    '''
+    Given a map from directory to the identifier for the device on which that directory resides,
+    return the directories as a sorted tuple with all duplicate child directories removed. For
+    instance, if paths is ('/foo', '/foo/bar'), return just: ('/foo',)
+
+    The one exception to this rule is if two paths are on different filesystems (devices). In that
+    case, they won't get de-duplicated in case they both need to be passed to Borg (e.g. the
+    location.one_file_system option is true).
 
     The idea is that if Borg is given a parent directory, then it doesn't also need to be given
     child directories, because it will naturally spider the contents of the parent directory. And
@@ -56,19 +70,23 @@ def deduplicate_directories(directories):
     Borg.
     '''
     deduplicated = set()
+    directories = sorted(directory_devices.keys())
 
-    for directory in sorted(directories):
+    for directory in directories:
-        # If the directory is "/", that contains all child directories, so we can early out.
+        deduplicated.add(directory)
-        if directory == os.path.sep:
+        parents = pathlib.PurePath(directory).parents
-            return (os.path.sep,)
 
-        # If no other directories are parents of current directory (even n levels up), then the
+        # If another directory in the given list is a parent of current directory (even n levels
-        # current directory isn't a duplicate.
+        # up) and both are on the same filesystem, then the current directory is a duplicate.
-        if not any(
+        for other_directory in directories:
-            pathlib.PurePath(other_directory) in pathlib.PurePath(directory).parents
+            for parent in parents:
-            for other_directory in directories
+                if (
-        ):
+                    pathlib.PurePath(other_directory) == parent
-            deduplicated.add(directory)
+                    and directory_devices[other_directory] == directory_devices[directory]
+                ):
+                    if directory in deduplicated:
+                        deduplicated.remove(directory)
+                    break
 
     return tuple(sorted(deduplicated))
 
@@ -179,9 +197,11 @@ def create_archive(
     create command while also triggering the given processes to produce output.
     '''
     sources = deduplicate_directories(
-        _expand_directories(
+        map_directories_to_devices(
-            location_config['source_directories']
+            _expand_directories(
-            + borgmatic_source_directories(location_config.get('borgmatic_source_directory'))
+                location_config['source_directories']
+                + borgmatic_source_directories(location_config.get('borgmatic_source_directory'))
+            )
         )
     )
 

borgmatic/borg/environment.py

@@ -19,7 +19,11 @@ DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE = {
 
 def initialize(storage_config):
     for option_name, environment_variable_name in OPTION_TO_ENVIRONMENT_VARIABLE.items():
-        value = storage_config.get(option_name)
+
+        # Options from borgmatic configuration take precedence over already set BORG_* environment
+        # variables.
+        value = storage_config.get(option_name) or os.environ.get(environment_variable_name)
+
         if value:
             os.environ[environment_variable_name] = value
         else:

borgmatic/commands/borgmatic.py

@@ -132,16 +132,6 @@ def run_configuration(config_filename, config, arguments):
 
     if not encountered_error:
         try:
-            if prune_create_or_check:
-                dispatch.call_hooks(
-                    'ping_monitor',
-                    hooks,
-                    config_filename,
-                    monitor.MONITOR_HOOK_NAMES,
-                    monitor.State.FINISH,
-                    monitoring_log_level,
-                    global_arguments.dry_run,
-                )
             if 'prune' in arguments:
                 command.execute_hook(
                     hooks.get('after_prune'),
@@ -174,6 +164,24 @@ def run_configuration(config_filename, config, arguments):
                     'post-check',
                     global_arguments.dry_run,
                 )
+            if prune_create_or_check:
+                dispatch.call_hooks(
+                    'ping_monitor',
+                    hooks,
+                    config_filename,
+                    monitor.MONITOR_HOOK_NAMES,
+                    monitor.State.FINISH,
+                    monitoring_log_level,
+                    global_arguments.dry_run,
+                )
+                dispatch.call_hooks(
+                    'destroy_monitor',
+                    hooks,
+                    config_filename,
+                    monitor.MONITOR_HOOK_NAMES,
+                    monitoring_log_level,
+                    global_arguments.dry_run,
+                )
         except (OSError, CalledProcessError) as error:
             if command.considered_soft_failure(config_filename, error):
                 return
@@ -185,15 +193,6 @@ def run_configuration(config_filename, config, arguments):
 
     if encountered_error and prune_create_or_check:
         try:
-            dispatch.call_hooks(
-                'ping_monitor',
-                hooks,
-                config_filename,
-                monitor.MONITOR_HOOK_NAMES,
-                monitor.State.FAIL,
-                monitoring_log_level,
-                global_arguments.dry_run,
-            )
             command.execute_hook(
                 hooks.get('on_error'),
                 hooks.get('umask'),
@@ -204,6 +203,23 @@ def run_configuration(config_filename, config, arguments):
                 error=encountered_error,
                 output=getattr(encountered_error, 'output', ''),
             )
+            dispatch.call_hooks(
+                'ping_monitor',
+                hooks,
+                config_filename,
+                monitor.MONITOR_HOOK_NAMES,
+                monitor.State.FAIL,
+                monitoring_log_level,
+                global_arguments.dry_run,
+            )
+            dispatch.call_hooks(
+                'destroy_monitor',
+                hooks,
+                config_filename,
+                monitor.MONITOR_HOOK_NAMES,
+                monitoring_log_level,
+                global_arguments.dry_run,
+            )
         except (OSError, CalledProcessError) as error:
             if command.considered_soft_failure(config_filename, error):
                 return

borgmatic/config/schema.yaml

@@ -3,9 +3,10 @@ version: 1
 map:
     location:
         desc: |
-            Where to look for files to backup, and where to store those backups. See
+            Where to look for files to backup, and where to store those backups.
-            https://borgbackup.readthedocs.io/en/stable/quickstart.html and
+            See https://borgbackup.readthedocs.io/en/stable/quickstart.html and
-            https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create for details.
+            https://borgbackup.readthedocs.io/en/stable/usage/create.html
+            for details.
         required: true
         map:
             source_directories:
@@ -13,7 +14,8 @@ map:
                 seq:
                     - type: str
                 desc: |
-                    List of source directories to backup (required). Globs and tildes are expanded.
+                    List of source directories to backup (required). Globs and
+                    tildes are expanded.
                 example:
                     - /home
                     - /etc
@@ -23,21 +25,25 @@ map:
                 seq:
                     - type: str
                 desc: |
-                    Paths to local or remote repositories (required). Tildes are expanded. Multiple
+                    Paths to local or remote repositories (required). Tildes are
-                    repositories are backed up to in sequence. See ssh_command for SSH options like
+                    expanded. Multiple repositories are backed up to in
-                    identity file or port.
+                    sequence. See ssh_command for SSH options like identity file
+                    or port.
                 example:
                     - user@backupserver:sourcehostname.borg
             one_file_system:
                 type: bool
                 desc: |
-                  Stay in same file system (do not cross mount points). Defaults to false. But when
+                    Stay in same file system (do not cross mount points).
-                  a database hook is used, the setting here is ignored and one_file_system is
+                    Defaults to false. But when a database hook is used, the
-                  considered true.
+                    setting here is ignored and one_file_system is considered
+                    true.
                 example: true
             numeric_owner:
                 type: bool
-                desc: Only store/extract numeric user and group identifiers. Defaults to false.
+                desc: |
+                    Only store/extract numeric user and group identifiers.
+                    Defaults to false.
                 example: true
             atime:
                 type: bool
@@ -49,27 +55,32 @@ map:
                 example: false
             birthtime:
                 type: bool
-                desc: Store birthtime (creation date) into archive. Defaults to true.
+                desc: |
+                    Store birthtime (creation date) into archive. Defaults to
+                    true.
                 example: false
             read_special:
                 type: bool
                 desc: |
-                    Use Borg's --read-special flag to allow backup of block and other special
+                    Use Borg's --read-special flag to allow backup of block and
-                    devices. Use with caution, as it will lead to problems if used when
+                    other special devices. Use with caution, as it will lead to
-                    backing up special devices such as /dev/zero. Defaults to false. But when a
+                    problems if used when backing up special devices such as
-                    database hook is used, the setting here is ignored and read_special is
+                    /dev/zero. Defaults to false. But when a database hook is
+                    used, the setting here is ignored and read_special is
                     considered true.
                 example: false
             bsd_flags:
                 type: bool
-                desc: Record bsdflags (e.g. NODUMP, IMMUTABLE) in archive. Defaults to true.
+                desc: |
+                    Record bsdflags (e.g. NODUMP, IMMUTABLE) in archive.
+                    Defaults to true.
                 example: true
             files_cache:
                 type: str
                 desc: |
                     Mode in which to operate the files cache. See
-                    https://borgbackup.readthedocs.io/en/stable/usage/create.html#description for
+                    http://borgbackup.readthedocs.io/en/stable/usage/create.html
-                    details. Defaults to "ctime,size,inode".
+                    for details. Defaults to "ctime,size,inode".
                 example: ctime,size,inode
             local_path:
                 type: str
@@ -83,9 +94,10 @@ map:
                 seq:
                     - type: str
                 desc: |
-                    Any paths matching these patterns are included/excluded from backups. Globs are
+                    Any paths matching these patterns are included/excluded from
-                    expanded. (Tildes are not.) Note that Borg considers this option experimental.
+                    backups. Globs are expanded. (Tildes are not.) Note that
-                    See the output of "borg help patterns" for more details. Quote any value if it
+                    Borg considers this option experimental. See the output of
+                    "borg help patterns" for more details. Quote any value if it
                     contains leading punctuation, so it parses correctly.
                 example:
                     - 'R /'
@@ -96,17 +108,19 @@ map:
                 seq:
                     - type: str
                 desc: |
-                    Read include/exclude patterns from one or more separate named files, one pattern
+                    Read include/exclude patterns from one or more separate
-                    per line. Note that Borg considers this option experimental. See the output of
+                    named files, one pattern per line. Note that Borg considers
-                    "borg help patterns" for more details.
+                    this option experimental. See the output of "borg help
+                    patterns" for more details.
                 example:
                     - /etc/borgmatic/patterns
             exclude_patterns:
                 seq:
                     - type: str
                 desc: |
-                    Any paths matching these patterns are excluded from backups. Globs and tildes
+                    Any paths matching these patterns are excluded from backups.
-                    are expanded. See the output of "borg help patterns" for more details.
+                    Globs and tildes are expanded. See the output of "borg help
+                    patterns" for more details.
                 example:
                     - '*.pyc'
                     - ~/*/.cache
@@ -115,29 +129,32 @@ map:
                 seq:
                     - type: str
                 desc: |
-                    Read exclude patterns from one or more separate named files, one pattern per
+                    Read exclude patterns from one or more separate named files,
-                    line. See the output of "borg help patterns" for more details.
+                    one pattern per line. See the output of "borg help patterns"
+                    for more details.
                 example:
                     - /etc/borgmatic/excludes
             exclude_caches:
                 type: bool
                 desc: |
                     Exclude directories that contain a CACHEDIR.TAG file. See
-                    http://www.brynosaurus.com/cachedir/spec.html for details. Defaults to false.
+                    http://www.brynosaurus.com/cachedir/spec.html for details.
+                    Defaults to false.
                 example: true
             exclude_if_present:
                 seq:
                     - type: str
                 desc: |
-                    Exclude directories that contain a file with the given filenames. Defaults to not
+                    Exclude directories that contain a file with the given
-                    set.
+                    filenames. Defaults to not set.
                 example:
                     - .nobackup
             keep_exclude_tags:
                 type: bool
                 desc: |
-                    If true, the exclude_if_present filename is included in backups. Defaults to
+                    If true, the exclude_if_present filename is included in
-                    false, meaning that the exclude_if_present filename is omitted from backups.
+                    backups. Defaults to false, meaning that the
+                    exclude_if_present filename is omitted from backups.
                 example: true
             exclude_nodump:
                 type: bool
@@ -147,90 +164,103 @@ map:
             borgmatic_source_directory:
                 type: str
                 desc: |
-                    Path for additional source files used for temporary internal state like
+                    Path for additional source files used for temporary internal
-                    borgmatic database dumps. Note that changing this path prevents "borgmatic
+                    state like borgmatic database dumps. Note that changing this
-                    restore" from finding any database dumps created before the change. Defaults
+                    path prevents "borgmatic restore" from finding any database
-                    to ~/.borgmatic
+                    dumps created before the change. Defaults to ~/.borgmatic
                 example: /tmp/borgmatic
     storage:
         desc: |
             Repository storage options. See
-            https://borgbackup.readthedocs.io/en/stable/usage.html#borg-create and
+            https://borgbackup.readthedocs.io/en/stable/usage/create.html and
-            https://borgbackup.readthedocs.io/en/stable/usage/general.html#environment-variables for
+            https://borgbackup.readthedocs.io/en/stable/usage/general.html for
             details.
         map:
             encryption_passcommand:
                 type: str
                 desc: |
-                    The standard output of this command is used to unlock the encryption key. Only
+                    The standard output of this command is used to unlock the
-                    use on repositories that were initialized with passcommand/repokey encryption.
+                    encryption key. Only use on repositories that were
-                    Note that if both encryption_passcommand and encryption_passphrase are set,
+                    initialized with passcommand/repokey encryption. Note that
-                    then encryption_passphrase takes precedence. Defaults to not set.
+                    if both encryption_passcommand and encryption_passphrase are
+                    set, then encryption_passphrase takes precedence. Defaults
+                    to not set.
                 example: "secret-tool lookup borg-repository repo-name"
             encryption_passphrase:
                 type: str
                 desc: |
-                    Passphrase to unlock the encryption key with. Only use on repositories that were
+                    Passphrase to unlock the encryption key with. Only use on
-                    initialized with passphrase/repokey encryption. Quote the value if it contains
+                    repositories that were initialized with passphrase/repokey
-                    punctuation, so it parses correctly. And backslash any quote or backslash
+                    encryption. Quote the value if it contains punctuation, so
+                    it parses correctly. And backslash any quote or backslash
                     literals as well. Defaults to not set.
                 example: "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
             checkpoint_interval:
                 type: int
                 desc: |
-                    Number of seconds between each checkpoint during a long-running backup. See
+                    Number of seconds between each checkpoint during a
-                    https://borgbackup.readthedocs.io/en/stable/faq.html#if-a-backup-stops-mid-way-does-the-already-backed-up-data-stay-there
+                    long-running backup. See
-                    for details. Defaults to checkpoints every 1800 seconds (30 minutes).
+                    https://borgbackup.readthedocs.io/en/stable/faq.html
+                    for details. Defaults to checkpoints every 1800 seconds (30
+                    minutes).
                 example: 1800
             chunker_params:
                 type: str
                 desc: |
-                    Specify the parameters passed to then chunker (CHUNK_MIN_EXP, CHUNK_MAX_EXP,
+                    Specify the parameters passed to then chunker
-                    HASH_MASK_BITS, HASH_WINDOW_SIZE). See https://borgbackup.readthedocs.io/en/stable/internals.html
+                    (CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS,
+                    HASH_WINDOW_SIZE). See
+                    https://borgbackup.readthedocs.io/en/stable/internals.html
                     for details. Defaults to "19,23,21,4095".
                 example: 19,23,21,4095
             compression:
                 type: str
                 desc: |
                     Type of compression to use when creating archives. See
-                    https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create for details.
+                    http://borgbackup.readthedocs.io/en/stable/usage/create.html
-                    Defaults to "lz4".
+                    for details. Defaults to "lz4".
                 example: lz4
             remote_rate_limit:
                 type: int
-                desc: Remote network upload rate limit in kiBytes/second. Defaults to unlimited.
+                desc: |
+                    Remote network upload rate limit in kiBytes/second. Defaults
+                    to unlimited.
                 example: 100
             ssh_command:
                 type: str
                 desc: |
-                    Command to use instead of "ssh". This can be used to specify ssh options.
+                    Command to use instead of "ssh". This can be used to specify
-                    Defaults to not set.
+                    ssh options.  Defaults to not set.
                 example: ssh -i /path/to/private/key
             borg_base_directory:
                 type: str
                 desc: |
-                    Base path used for various Borg directories. Defaults to $HOME, ~$USER, or ~.
+                    Base path used for various Borg directories. Defaults to
-                    See https://borgbackup.readthedocs.io/en/stable/usage/general.html#environment-variables for details.
+                    $HOME, ~$USER, or ~.
                 example: /path/to/base
             borg_config_directory:
                 type: str
                 desc: |
-                    Path for Borg configuration files. Defaults to $borg_base_directory/.config/borg
+                    Path for Borg configuration files. Defaults to
+                    $borg_base_directory/.config/borg
                 example: /path/to/base/config
             borg_cache_directory:
                 type: str
                 desc: |
-                    Path for Borg cache files. Defaults to $borg_base_directory/.cache/borg
+                    Path for Borg cache files. Defaults to
+                    $borg_base_directory/.cache/borg
                 example: /path/to/base/cache
             borg_security_directory:
                 type: str
                 desc: |
-                    Path for Borg security and encryption nonce files. Defaults to $borg_base_directory/.config/borg/security
+                    Path for Borg security and encryption nonce files. Defaults
+                    to $borg_base_directory/.config/borg/security
                 example: /path/to/base/config/security
             borg_keys_directory:
                 type: str
                 desc: |
-                    Path for Borg encryption key files. Defaults to $borg_base_directory/.config/borg/keys
+                    Path for Borg encryption key files. Defaults to
+                    $borg_base_directory/.config/borg/keys
                 example: /path/to/base/config/keys
             umask:
                 type: scalar
@@ -238,58 +268,69 @@ map:
                 example: 0077
             lock_wait:
                 type: int
-                desc: Maximum seconds to wait for acquiring a repository/cache lock. Defaults to 1.
+                desc: |
+                    Maximum seconds to wait for acquiring a repository/cache
+                    lock. Defaults to 1.
                 example: 5
             archive_name_format:
                 type: str
                 desc: |
-                    Name of the archive. Borg placeholders can be used. See the output of
+                    Name of the archive. Borg placeholders can be used. See the
-                    "borg help placeholders" for details. Defaults to
+                    output of "borg help placeholders" for details. Defaults to
-                    "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". If you specify this option, you must
+                    "{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}". If you specify this
-                    also specify a prefix in the retention section to avoid accidental pruning of
+                    option, you must also specify a prefix in the retention
-                    archives with a different archive name format. And you should also specify a
+                    section to avoid accidental pruning of archives with a
+                    different archive name format. And you should also specify a
                     prefix in the consistency section as well.
                 example: "{hostname}-documents-{now}"
             relocated_repo_access_is_ok:
                 type: bool
-                desc: Bypass Borg error about a repository that has been moved. Defaults to false.
+                desc: |
+                    Bypass Borg error about a repository that has been moved.
+                    Defaults to false.
                 example: true
             unknown_unencrypted_repo_access_is_ok:
                 type: bool
                 desc: |
-                    Bypass Borg error about a previously unknown unencrypted repository. Defaults to
+                    Bypass Borg error about a previously unknown unencrypted
-                    false.
+                    repository. Defaults to false.
                 example: true
             extra_borg_options:
                 map:
                     init:
                         type: str
-                        desc: Extra command-line options to pass to "borg init".
+                        desc: |
+                            Extra command-line options to pass to "borg init".
                         example: "--make-parent-dirs"
                     prune:
                         type: str
-                        desc: Extra command-line options to pass to "borg prune".
+                        desc: |
+                            Extra command-line options to pass to "borg prune".
                         example: "--save-space"
                     create:
                         type: str
-                        desc: Extra command-line options to pass to "borg create".
+                        desc: |
+                            Extra command-line options to pass to "borg create".
                         example: "--no-files-cache"
                     check:
                         type: str
-                        desc: Extra command-line options to pass to "borg check".
+                        desc: |
+                            Extra command-line options to pass to "borg check".
                         example: "--save-space"
                 desc: |
-                    Additional options to pass directly to particular Borg commands, handy for Borg
+                    Additional options to pass directly to particular Borg
-                    options that borgmatic does not yet support natively. Note that borgmatic does
+                    commands, handy for Borg options that borgmatic does not yet
-                    not perform any validation on these options. Running borgmatic with
+                    support natively. Note that borgmatic does not perform any
-                    "--verbosity 2" shows the exact Borg command-line invocation.
+                    validation on these options. Running borgmatic with
+                    "--verbosity 2" shows the exact Borg command-line
+                    invocation.
     retention:
         desc: |
             Retention policy for how many backups to keep in each category. See
-            https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details.
+            https://borgbackup.readthedocs.io/en/stable/usage/prune.html for
-            At least one of the "keep" options is required for pruning to work. See
+            details. At least one of the "keep" options is required for pruning
-            https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/
+            to work. See borgmatic documentation if you'd like to skip pruning
-            if you'd like to skip pruning entirely.
+            entirely.
         map:
             keep_within:
                 type: str
@@ -326,27 +367,37 @@ map:
             prefix:
                 type: str
                 desc: |
-                    When pruning, only consider archive names starting with this prefix.
+                    When pruning, only consider archive names starting with this
-                    Borg placeholders can be used. See the output of "borg help placeholders" for
+                    prefix.  Borg placeholders can be used. See the output of
-                    details. Defaults to "{hostname}-". Use an empty value to disable the default.
+                    "borg help placeholders" for details. Defaults to
+                    "{hostname}-". Use an empty value to disable the default.
                 example: sourcehostname
     consistency:
         desc: |
             Consistency checks to run after backups. See
-            https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check and
+            https://borgbackup.readthedocs.io/en/stable/usage/check.html and
-            https://borgbackup.readthedocs.org/en/stable/usage.html#borg-extract for details.
+            https://borgbackup.readthedocs.io/en/stable/usage/extract.html for
+            details.
         map:
             checks:
                 seq:
                     - type: str
-                      enum: ['repository', 'archives', 'data', 'extract', 'disabled']
+                      enum: [
+                          'repository',
+                          'archives',
+                          'data',
+                          'extract',
+                          'disabled'
+                      ]
                       unique: true
                 desc: |
-                    List of one or more consistency checks to run: "repository", "archives", "data",
+                    List of one or more consistency checks to run: "repository",
-                    and/or "extract". Defaults to "repository" and "archives". Set to "disabled" to
+                    "archives", "data", and/or "extract". Defaults to
-                    disable all consistency checks. "repository" checks the consistency of the
+                    "repository" and "archives". Set to "disabled" to disable
-                    repository, "archives" checks all of the archives, "data" verifies the integrity
+                    all consistency checks. "repository" checks the consistency
-                    of the data within the archives, and "extract" does an extraction dry-run of the
+                    of the repository, "archives" checks all of the archives,
+                    "data" verifies the integrity of the data within the
+                    archives, and "extract" does an extraction dry-run of the
                     most recent archive. Note that "data" implies "archives".
                 example:
                     - repository
@@ -355,24 +406,29 @@ map:
                 seq:
                     - type: str
                 desc: |
-                    Paths to a subset of the repositories in the location section on which to run
+                    Paths to a subset of the repositories in the location
-                    consistency checks. Handy in case some of your repositories are very large, and
+                    section on which to run consistency checks. Handy in case
-                    so running consistency checks on them would take too long. Defaults to running
+                    some of your repositories are very large, and so running
-                    consistency checks on all repositories configured in the location section.
+                    consistency checks on them would take too long. Defaults to
+                    running consistency checks on all repositories configured in
+                    the location section.
                 example:
                     - user@backupserver:sourcehostname.borg
             check_last:
                 type: int
-                desc: Restrict the number of checked archives to the last n. Applies only to the
+                desc: |
-                      "archives" check. Defaults to checking all archives.
+                    Restrict the number of checked archives to the last n.
+                    Applies only to the "archives" check. Defaults to checking
+                    all archives.
                 example: 3
             prefix:
                 type: str
                 desc: |
-                    When performing the "archives" check, only consider archive names starting with
+                    When performing the "archives" check, only consider archive
-                    this prefix. Borg placeholders can be used. See the output of
+                    names starting with this prefix. Borg placeholders can be
-                    "borg help placeholders" for details. Defaults to "{hostname}-". Use an empty
+                    used. See the output of "borg help placeholders" for
-                    value to disable the default.
+                    details. Defaults to "{hostname}-". Use an empty value to
+                    disable the default.
                 example: sourcehostname
     output:
         desc: |
@@ -381,72 +437,73 @@ map:
             color:
                 type: bool
                 desc: |
-                    Apply color to console output. Can be overridden with --no-color command-line
+                    Apply color to console output. Can be overridden with
-                    flag. Defaults to true.
+                    --no-color command-line flag. Defaults to true.
                 example: false
     hooks:
         desc: |
-            Shell commands, scripts, or integrations to execute at various points during a borgmatic
+            Shell commands, scripts, or integrations to execute at various
-            run. IMPORTANT: All provided commands and scripts are executed with user permissions of
+            points during a borgmatic run. IMPORTANT: All provided commands and
-            borgmatic. Do not forget to set secure permissions on this configuration file (chmod
+            scripts are executed with user permissions of borgmatic. Do not
-            0600) as well as on any script called from a hook (chmod 0700) to prevent potential
+            forget to set secure permissions on this configuration file (chmod
-            shell injection or privilege escalation.
+            0600) as well as on any script called from a hook (chmod 0700) to
+            prevent potential shell injection or privilege escalation.
         map:
             before_backup:
                 seq:
                     - type: str
                 desc: |
-                    List of one or more shell commands or scripts to execute before creating a
+                    List of one or more shell commands or scripts to execute
-                    backup, run once per configuration file.
+                    before creating a backup, run once per configuration file.
                 example:
                     - echo "Starting a backup."
             before_prune:
                 seq:
                     - type: str
                 desc: |
-                    List of one or more shell commands or scripts to execute before pruning, run
+                    List of one or more shell commands or scripts to execute
-                    once per configuration file.
+                    before pruning, run once per configuration file.
                 example:
                     - echo "Starting pruning."
             before_check:
                 seq:
                     - type: str
                 desc: |
-                    List of one or more shell commands or scripts to execute before consistency
+                    List of one or more shell commands or scripts to execute
-                    checks, run once per configuration file.
+                    before consistency checks, run once per configuration file.
                 example:
                     - echo "Starting checks."
             after_backup:
                 seq:
                     - type: str
                 desc: |
-                    List of one or more shell commands or scripts to execute after creating a
+                    List of one or more shell commands or scripts to execute
-                    backup, run once per configuration file.
+                    after creating a backup, run once per configuration file.
                 example:
                     - echo "Finished a backup."
             after_prune:
                 seq:
                     - type: str
                 desc: |
-                    List of one or more shell commands or scripts to execute after pruning, run once
+                    List of one or more shell commands or scripts to execute
-                    per configuration file.
+                    after pruning, run once per configuration file.
                 example:
                     - echo "Finished pruning."
             after_check:
                 seq:
                     - type: str
                 desc: |
-                    List of one or more shell commands or scripts to execute after consistency
+                    List of one or more shell commands or scripts to execute
-                    checks, run once per configuration file.
+                    after consistency checks, run once per configuration file.
                 example:
                     - echo "Finished checks."
             on_error:
                 seq:
                     - type: str
                 desc: |
-                    List of one or more shell commands or scripts to execute when an exception
+                    List of one or more shell commands or scripts to execute
-                    occurs during a "prune", "create", or "check" action or an associated
+                    when an exception occurs during a "prune", "create", or
-                    before/after hook.
+                    "check" action or an associated before/after hook.
                 example:
                     - echo "Error during prune/create/check."
             postgresql_databases:
@@ -456,16 +513,17 @@ map:
                             required: true
                             type: str
                             desc: |
-                                Database name (required if using this hook). Or "all" to dump all
+                                Database name (required if using this hook). Or
-                                databases on the host. Note that using this database hook implicitly
+                                "all" to dump all databases on the host. Note
-                                enables both read_special and one_file_system (see above) to support
+                                that using this database hook implicitly enables
-                                dump and restore streaming.
+                                both read_special and one_file_system (see
+                                above) to support dump and restore streaming.
                             example: users
                         hostname:
                             type: str
                             desc: |
-                                Database hostname to connect to. Defaults to connecting via local
+                                Database hostname to connect to. Defaults to
-                                Unix socket.
+                                connecting via local Unix socket.
                             example: database.example.org
                         port:
                             type: int
@@ -474,39 +532,78 @@ map:
                         username:
                             type: str
                             desc: |
-                                Username with which to connect to the database. Defaults to the
+                                Username with which to connect to the database.
-                                username of the current user. You probably want to specify the
+                                Defaults to the username of the current user.
-                                "postgres" superuser here when the database name is "all".
+                                You probably want to specify the "postgres"
+                                superuser here when the database name is "all".
                             example: dbuser
                         password:
                             type: str
                             desc: |
-                                Password with which to connect to the database. Omitting a password
+                                Password with which to connect to the database.
-                                will only work if PostgreSQL is configured to trust the configured
+                                Omitting a password will only work if PostgreSQL
-                                username without a password, or you create a ~/.pgpass file.
+                                is configured to trust the configured username
+                                without a password, or you create a ~/.pgpass
+                                file.
                             example: trustsome1
                         format:
                             type: str
                             enum: ['plain', 'custom', 'directory', 'tar']
                             desc: |
-                                Database dump output format. One of "plain", "custom", "directory",
+                                Database dump output format. One of "plain",
-                                or "tar". Defaults to "custom" (unlike raw pg_dump). See
+                                "custom", "directory", or "tar". Defaults to
-                                https://www.postgresql.org/docs/current/app-pgdump.html for details.
+                                "custom" (unlike raw pg_dump). See pg_dump
-                                Note that format is ignored when the database name is "all".
+                                documentation for details. Note that format is
+                                ignored when the database name is "all".
                             example: directory
+                        ssl_mode:
+                            type: str
+                            enum: ['disable', 'allow', 'prefer',
+                                  'require', 'verify-ca', 'verify-full']
+                            desc: |
+                                SSL mode to use to connect to the database
+                                server. One of "disable", "allow", "prefer",
+                                "require", "verify-ca" or "verify-full".
+                                Defaults to "disable".
+                            example: require
+                        ssl_cert:
+                            type: str
+                            desc: |
+                                Path to a client certificate.
+                            example: "/root/.postgresql/postgresql.crt"
+                        ssl_key:
+                            type: str
+                            desc: |
+                                Path to a private client key.
+                            example: "/root/.postgresql/postgresql.key"
+                        ssl_root_cert:
+                            type: str
+                            desc: |
+                                Path to a root certificate containing a list of
+                                trusted certificate authorities.
+                            example: "/root/.postgresql/root.crt"
+                        ssl_crl:
+                            type: str
+                            desc: |
+                                Path to a certificate revocation list.
+                            example: "/root/.postgresql/root.crl"
                         options:
                             type: str
                             desc: |
-                                Additional pg_dump/pg_dumpall options to pass directly to the dump
+                                Additional pg_dump/pg_dumpall options to pass
-                                command, without performing any validation on them. See
+                                directly to the dump command, without performing
-                                https://www.postgresql.org/docs/current/app-pgdump.html for details.
+                                any validation on them. See pg_dump
+                                documentation for details.
                             example: --role=someone
                 desc: |
-                    List of one or more PostgreSQL databases to dump before creating a backup,
+                    List of one or more PostgreSQL databases to dump before
-                    run once per configuration file. The database dumps are added to your source
+                    creating a backup, run once per configuration file. The
-                    directories at runtime, backed up, and then removed afterwards. Requires
+                    database dumps are added to your source directories at
+                    runtime, backed up, and removed afterwards. Requires
                     pg_dump/pg_dumpall/pg_restore commands. See
-                    https://www.postgresql.org/docs/current/app-pgdump.html for details.
+                    https://www.postgresql.org/docs/current/app-pgdump.html and
+                    https://www.postgresql.org/docs/current/libpq-ssl.html for
+                    details.
             mysql_databases:
                 seq:
                     - map:
@@ -514,16 +611,17 @@ map:
                             required: true
                             type: str
                             desc: |
-                                Database name (required if using this hook). Or "all" to dump all
+                                Database name (required if using this hook). Or
-                                databases on the host. Note that using this database hook implicitly
+                                "all" to dump all databases on the host. Note
-                                enables both read_special and one_file_system (see above) to support
+                                that using this database hook implicitly enables
-                                dump and restore streaming.
+                                both read_special and one_file_system (see
+                                above) to support dump and restore streaming.
                             example: users
                         hostname:
                             type: str
                             desc: |
-                                Database hostname to connect to. Defaults to connecting via local
+                                Database hostname to connect to. Defaults to
-                                Unix socket.
+                                connecting via local Unix socket.
                             example: database.example.org
                         port:
                             type: int
@@ -532,87 +630,92 @@ map:
                         username:
                             type: str
                             desc: |
-                                Username with which to connect to the database. Defaults to the
+                                Username with which to connect to the database.
-                                username of the current user.
+                                Defaults to the username of the current user.
                             example: dbuser
                         password:
                             type: str
                             desc: |
-                                Password with which to connect to the database. Omitting a password
+                                Password with which to connect to the database.
-                                will only work if MySQL is configured to trust the configured
+                                Omitting a password will only work if MySQL is
-                                username without a password.
+                                configured to trust the configured username
+                                without a password.
                             example: trustsome1
                         options:
                             type: str
                             desc: |
-                                Additional mysqldump options to pass directly to the dump command,
+                                Additional mysqldump options to pass directly to
-                                without performing any validation on them. See
+                                the dump command, without performing any
-                                https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html or
+                                validation on them. See mysqldump documentation
-                                https://mariadb.com/kb/en/library/mysqldump/ for details.
+                                for details.
                             example: --skip-comments
                 desc: |
-                    List of one or more MySQL/MariaDB databases to dump before creating a backup,
+                    List of one or more MySQL/MariaDB databases to dump before
-                    run once per configuration file. The database dumps are added to your source
+                    creating a backup, run once per configuration file. The
-                    directories at runtime, backed up, and then removed afterwards. Requires
+                    database dumps are added to your source directories at
+                    runtime, backed up, and removed afterwards. Requires
                     mysqldump/mysql commands (from either MySQL or MariaDB). See
                     https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html or
                     https://mariadb.com/kb/en/library/mysqldump/ for details.
             healthchecks:
                 type: str
                 desc: |
-                    Healthchecks ping URL or UUID to notify when a backup begins, ends, or errors.
+                    Healthchecks ping URL or UUID to notify when a backup
-                    Create an account at https://healthchecks.io if you'd like to use this service.
+                    begins, ends, or errors. Create an account at
-                    See
+                    https://healthchecks.io if you'd like to use this service.
-                    https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook
+                    See borgmatic monitoring documentation for details.
-                    for details.
                 example:
                     https://hc-ping.com/your-uuid-here
             cronitor:
                 type: str
                 desc: |
-                    Cronitor ping URL to notify when a backup begins, ends, or errors. Create an
+                    Cronitor ping URL to notify when a backup begins, ends, or
-                    account at https://cronitor.io if you'd like to use this service. See
+                    errors. Create an account at https://cronitor.io if you'd
-                    https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook
+                    like to use this service. See borgmatic monitoring
-                    for details.
+                    documentation for details.
                 example:
                     https://cronitor.link/d3x0c1
             pagerduty:
                 type: str
                 desc: |
-                    PagerDuty integration key used to notify PagerDuty when a backup errors. Create
+                    PagerDuty integration key used to notify PagerDuty when a
-                    an account at https://www.pagerduty.com/ if you'd like to use this service. See
+                    backup errors. Create an account at
-                    https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook
+                    https://www.pagerduty.com/ if you'd like to use this
-                    for details.
+                    service. See borgmatic monitoring documentation for details.
                 example:
                     a177cad45bd374409f78906a810a3074
             cronhub:
                 type: str
                 desc: |
-                    Cronhub ping URL to notify when a backup begins, ends, or errors. Create an
+                    Cronhub ping URL to notify when a backup begins, ends, or
-                    account at https://cronhub.io if you'd like to use this service. See
+                    errors. Create an account at https://cronhub.io if you'd
-                    https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronhub-hook for
+                    like to use this service. See borgmatic monitoring
-                    details.
+                    documentation for details.
                 example:
-                    https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d031
+                    https://cronhub.io/start/1f5e3410-254c-11e8-b61d-55875966d01
             before_everything:
                 seq:
                     - type: str
                 desc: |
-                    List of one or more shell commands or scripts to execute before running all
+                    List of one or more shell commands or scripts to execute
-                    actions (if one of them is "create"). These are collected from all configuration
+                    before running all actions (if one of them is "create").
-                    files and then run once before all of them (prior to all actions).
+                    These are collected from all configuration files and then
+                    run once before all of them (prior to all actions).
                 example:
                     - echo "Starting actions."
             after_everything:
                 seq:
                     - type: str
                 desc: |
-                    List of one or more shell commands or scripts to execute after running all
+                    List of one or more shell commands or scripts to execute
-                    actions (if one of them is "create"). These are collected from all configuration
+                    after running all actions (if one of them is "create").
-                    files and then run once before all of them (prior to all actions).
+                    These are collected from all configuration files and then
+                    run once before all of them (prior to all actions).
                 example:
                     - echo "Completed actions."
             umask:
                 type: scalar
-                desc: Umask used when executing hooks. Defaults to the umask that borgmatic is run with.
+                desc: |
+                    Umask used when executing hooks. Defaults to the umask that
+                    borgmatic is run with.
                 example: 0077

borgmatic/hooks/healthchecks.py

@@ -107,3 +107,15 @@ def ping_monitor(ping_url_or_uuid, config_filename, state, monitoring_log_level,
     if not dry_run:
         logging.getLogger('urllib3').setLevel(logging.ERROR)
         requests.post(ping_url, data=payload.encode('utf-8'))
+
+
+def destroy_monitor(ping_url_or_uuid, config_filename, monitoring_log_level, dry_run):
+    '''
+    Remove the monitor handler that was added to the root logger. This prevents the handler from
+    getting reused by other instances of this monitor.
+    '''
+    logger = logging.getLogger()
+
+    for handler in tuple(logger.handlers):
+        if isinstance(handler, Forgetful_buffering_handler):
+            logger.removeHandler(handler)

borgmatic/hooks/postgresql.py

@@ -15,6 +15,25 @@ def make_dump_path(location_config):  # pragma: no cover
     )
 
 
+def make_extra_environment(database):
+    '''
+    Make the extra_environment dict from the given database configuration.
+    '''
+    extra = dict()
+    if 'password' in database:
+        extra['PGPASSWORD'] = database['password']
+    extra['PGSSLMODE'] = database.get('ssl_mode', 'disable')
+    if 'ssl_cert' in database:
+        extra['PGSSLCERT'] = database['ssl_cert']
+    if 'ssl_key' in database:
+        extra['PGSSLKEY'] = database['ssl_key']
+    if 'ssl_root_cert' in database:
+        extra['PGSSLROOTCERT'] = database['ssl_root_cert']
+    if 'ssl_crl' in database:
+        extra['PGSSLCRL'] = database['ssl_crl']
+    return extra
+
+
 def dump_databases(databases, log_prefix, location_config, dry_run):
     '''
     Dump the given PostgreSQL databases to a named pipe. The databases are supplied as a sequence of
@@ -56,7 +75,7 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
             # format in a particular, a named destination is required, and redirection doesn't work.
             + (('>', dump_filename) if dump_format != 'directory' else ())
         )
-        extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
+        extra_environment = make_extra_environment(database)
 
         logger.debug(
             '{}: Dumping PostgreSQL database {} to {}{}'.format(
@@ -141,7 +160,7 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
         + (('--username', database['username']) if 'username' in database else ())
         + (() if extract_process else (dump_filename,))
     )
-    extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
+    extra_environment = make_extra_environment(database)
 
     logger.debug(
         '{}: Restoring PostgreSQL database {}{}'.format(log_prefix, database['name'], dry_run_label)

docs/_includes/index.css

@@ -181,7 +181,7 @@ pre {
 	padding: .5em;
 	margin: 1em -.5em 2em -.5em;
 	overflow-x: auto;
-	background-color: #eee;
+	background-color: #fafafa;
 	font-size: 0.75em; /* 12px /16 */
 }
 pre,
@@ -194,7 +194,7 @@ code {
 	-webkit-hyphens: manual;
 	-moz-hyphens: manual;
 	hyphens: manual;
-	background-color: #efefef;
+	background-color: #fafafa;
 }
 pre + pre[class*="language-"] {
 	margin-top: 1em;

docs/_includes/prism-theme.css

@@ -3,9 +3,12 @@
  * Based on dabblet (http://dabblet.com)
  * @author Lea Verou
  */
+/*
+ * Modified with an approximation of the One Light syntax highlighting theme.
+ */
 code[class*="language-"],
 pre[class*="language-"] {
-  color: #ABB2BF;
+  color: #494b53;
   background: none;
   font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace;
   text-align: left;
@@ -26,13 +29,15 @@ pre[class*="language-"] {
 pre[class*="language-"]::-moz-selection, pre[class*="language-"] ::-moz-selection,
 code[class*="language-"]::-moz-selection, code[class*="language-"] ::-moz-selection {
   text-shadow: none;
-  background: #383e49;
+  color: #232324;
+  background: #dbdbdc;
 }
 
 pre[class*="language-"]::selection, pre[class*="language-"] ::selection,
 code[class*="language-"]::selection, code[class*="language-"] ::selection {
   text-shadow: none;
-  background: #9aa2b1;
+  color: #232324;
+  background: #dbdbdc;
 }
 
 @media print {
@@ -50,7 +55,7 @@ pre[class*="language-"] {
 
 :not(pre) > code[class*="language-"],
 pre[class*="language-"] {
-  background: #282c34;
+  background: #fafafa;
 }
 
 /* Inline code */
@@ -64,16 +69,16 @@ pre[class*="language-"] {
 .token.prolog,
 .token.doctype,
 .token.cdata {
-  color: #5C6370;
+  color: #505157;
 }
 
 .token.punctuation {
-  color: #abb2bf;
+  color: #526fff;
 }
 
 .token.selector,
 .token.tag {
-  color: #e06c75;
+  color: none;
 }
 
 .token.property,
@@ -83,7 +88,7 @@ pre[class*="language-"] {
 .token.symbol,
 .token.attr-name,
 .token.deleted {
-  color: #d19a66;
+  color: #986801;
 }
 
 .token.string,
@@ -91,7 +96,7 @@ pre[class*="language-"] {
 .token.attr-value,
 .token.builtin,
 .token.inserted {
-  color: #98c379;
+  color: #50a14f;
 }
 
 .token.operator,
@@ -99,22 +104,22 @@ pre[class*="language-"] {
 .token.url,
 .language-css .token.string,
 .style .token.string {
-  color: #56b6c2;
+  color: #526fff;
 }
 
 .token.atrule,
 .token.keyword {
-  color: #e06c75;
+  color: #e45649;
 }
 
 .token.function {
-  color: #61afef;
+  color: #4078f2;
 }
 
 .token.regex,
 .token.important,
 .token.variable {
-  color: #c678dd;
+  color: #e45649;
 }
 
 .token.important,

docs/how-to/monitor-your-backups.md

@@ -123,13 +123,13 @@ hooks</a> run, borgmatic lets Healthchecks know that it has started if any of
 the `prune`, `create`, or `check` actions are run.
 
 Then, if the actions complete successfully, borgmatic notifies Healthchecks of
-the success before the `after_backup` hooks run, and includes borgmatic logs in
+the success after the `after_backup` hooks run, and includes borgmatic logs in
 the payload data sent to Healthchecks. This means that borgmatic logs show up
 in the Healthchecks UI, although be aware that Healthchecks currently has a
 10-kilobyte limit for the logs in each ping.
 
 If an error occurs during any action or hook, borgmatic notifies Healthchecks
-before the `on_error` hooks run, also tacking on logs including the error
+after the `on_error` hooks run, also tacking on logs including the error
 itself. But the logs are only included for errors that occur when a `prune`,
 `create`, or `check` action is run.
 
@@ -161,9 +161,9 @@ begins, ends, or errors. Specifically, after the <a
 href="https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/">`before_backup`
 hooks</a> run, borgmatic lets Cronitor know that it has started if any of the
 `prune`, `create`, or `check` actions are run. Then, if the actions complete
-successfully, borgmatic notifies Cronitor of the success before the
+successfully, borgmatic notifies Cronitor of the success after the
 `after_backup` hooks run. And if an error occurs during any action or hook,
-borgmatic notifies Cronitor before the `on_error` hooks run.
+borgmatic notifies Cronitor after the `on_error` hooks run.
 
 You can configure Cronitor to notify you by a [variety of
 mechanisms](https://cronitor.io/docs/cron-job-notifications) when backups fail
@@ -189,9 +189,9 @@ begins, ends, or errors. Specifically, after the <a
 href="https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/">`before_backup`
 hooks</a> run, borgmatic lets Cronhub know that it has started if any of the
 `prune`, `create`, or `check` actions are run. Then, if the actions complete
-successfully, borgmatic notifies Cronhub of the success before the
+successfully, borgmatic notifies Cronhub of the success after the
 `after_backup` hooks run. And if an error occurs during any action or hook,
-borgmatic notifies Cronhub before the `on_error` hooks run.
+borgmatic notifies Cronhub after the `on_error` hooks run.
 
 Note that even though you configure borgmatic with the "start" variant of the
 ping URL, borgmatic substitutes the correct state into the URL when pinging

docs/how-to/set-up-backups.md

@@ -8,9 +8,10 @@ these instructions install and run borgmatic as root. If you don't need to
 backup such files, then you are welcome to install and run borgmatic as a
 non-root user.
 
-First, [install
+First, manually [install
 Borg](https://borgbackup.readthedocs.io/en/stable/installation.html), at least
-version 1.1.
+version 1.1. borgmatic does not install Borg automatically so as to avoid
+conflicts with existing Borg installations.
 
 Then, download and install borgmatic as a [user site
 installation](https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site)

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.5.6'
+VERSION = '1.5.7.dev0'
 
 
 setup(

tests/integration/config/test_schema.py

@@ -0,0 +1,8 @@
+MAXIMUM_LINE_LENGTH = 80
+
+
+def test_schema_line_length_stays_under_limit():
+    schema_file = open('borgmatic/config/schema.yaml')
+
+    for line in schema_file.readlines():
+        assert len(line.rstrip('\n')) <= MAXIMUM_LINE_LENGTH

tests/integration/hooks/test_healthchecks.py

@@ -0,0 +1,24 @@
+import logging
+
+from flexmock import flexmock
+
+from borgmatic.hooks import healthchecks as module
+
+
+def test_destroy_monitor_removes_healthchecks_handler():
+    logger = logging.getLogger()
+    original_handlers = list(logger.handlers)
+    logger.addHandler(module.Forgetful_buffering_handler(byte_capacity=100, log_level=1))
+
+    module.destroy_monitor(flexmock(), flexmock(), flexmock(), flexmock())
+
+    assert logger.handlers == original_handlers
+
+
+def test_destroy_monitor_without_healthchecks_handler_does_not_raise():
+    logger = logging.getLogger()
+    original_handlers = list(logger.handlers)
+
+    module.destroy_monitor(flexmock(), flexmock(), flexmock(), flexmock())
+
+    assert logger.handlers == original_handlers

tests/unit/borg/test_create.py

@@ -63,20 +63,25 @@ def test_expand_home_directories_considers_none_as_no_directories():
 @pytest.mark.parametrize(
     'directories,expected_directories',
     (
-        (('/', '/root'), ('/',)),
+        ({'/': 1, '/root': 1}, ('/',)),
-        (('/', '/root/'), ('/',)),
+        ({'/': 1, '/root/': 1}, ('/',)),
-        (('/root', '/'), ('/',)),
+        ({'/': 1, '/root': 2}, ('/', '/root')),
-        (('/root', '/root/foo'), ('/root',)),
+        ({'/root': 1, '/': 1}, ('/',)),
-        (('/root/', '/root/foo'), ('/root/',)),
+        ({'/root': 1, '/root/foo': 1}, ('/root',)),
-        (('/root', '/root/foo/'), ('/root',)),
+        ({'/root/': 1, '/root/foo': 1}, ('/root/',)),
-        (('/root/foo', '/root'), ('/root',)),
+        ({'/root': 1, '/root/foo/': 1}, ('/root',)),
-        (('/root', '/etc', '/root/foo/bar'), ('/etc', '/root')),
+        ({'/root': 1, '/root/foo': 2}, ('/root', '/root/foo')),
-        (('/root', '/root/foo', '/root/foo/bar'), ('/root',)),
+        ({'/root/foo': 1, '/root': 1}, ('/root',)),
-        (('/dup', '/dup'), ('/dup',)),
+        ({'/root': 1, '/etc': 1, '/root/foo/bar': 1}, ('/etc', '/root')),
-        (('/foo', '/bar'), ('/bar', '/foo')),
+        ({'/root': 1, '/root/foo': 1, '/root/foo/bar': 1}, ('/root',)),
+        ({'/dup': 1, '/dup': 1}, ('/dup',)),
+        ({'/foo': 1, '/bar': 1}, ('/bar', '/foo')),
+        ({'/foo': 1, '/bar': 2}, ('/bar', '/foo')),
     ),
 )
-def test_deduplicate_directories_removes_child_paths(directories, expected_directories):
+def test_deduplicate_directories_removes_child_paths_on_the_same_filesystem(
+    directories, expected_directories
+):
     assert module.deduplicate_directories(directories) == expected_directories
 
 
@@ -235,6 +240,7 @@ ARCHIVE_WITH_PATHS = ('repo::{}'.format(DEFAULT_ARCHIVE_NAME), 'foo', 'bar')
 def test_create_archive_calls_borg_with_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -263,6 +269,7 @@ def test_create_archive_with_patterns_calls_borg_with_patterns():
     pattern_flags = ('--patterns-from', 'patterns')
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(
@@ -293,6 +300,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
     exclude_flags = ('--exclude-from', 'excludes')
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(('exclude',))
     flexmock(module).should_receive('_write_pattern_file').and_return(None).and_return(
@@ -322,6 +330,7 @@ def test_create_archive_with_exclude_patterns_calls_borg_with_excludes():
 def test_create_archive_with_log_info_calls_borg_with_info_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -351,6 +360,7 @@ def test_create_archive_with_log_info_calls_borg_with_info_parameter():
 def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -381,6 +391,7 @@ def test_create_archive_with_log_info_and_json_suppresses_most_borg_output():
 def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -409,6 +420,7 @@ def test_create_archive_with_log_debug_calls_borg_with_debug_parameter():
 def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -438,6 +450,7 @@ def test_create_archive_with_log_debug_and_json_suppresses_most_borg_output():
 def test_create_archive_with_dry_run_calls_borg_with_dry_run_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -468,6 +481,7 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete
     # https://borgbackup.readthedocs.io/en/stable/usage/create.html#description
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -498,6 +512,7 @@ def test_create_archive_with_stats_and_dry_run_calls_borg_without_stats_paramete
 def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_interval_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -525,6 +540,7 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
 def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -552,6 +568,7 @@ def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_param
 def test_create_archive_with_compression_calls_borg_with_compression_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -579,6 +596,7 @@ def test_create_archive_with_compression_calls_borg_with_compression_parameters(
 def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -606,6 +624,7 @@ def test_create_archive_with_remote_rate_limit_calls_borg_with_remote_ratelimit_
 def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -634,6 +653,7 @@ def test_create_archive_with_one_file_system_calls_borg_with_one_file_system_par
 def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -662,6 +682,7 @@ def test_create_archive_with_numeric_owner_calls_borg_with_numeric_owner_paramet
 def test_create_archive_with_read_special_calls_borg_with_read_special_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -691,6 +712,7 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
 def test_create_archive_with_option_true_calls_borg_without_corresponding_parameter(option_name):
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -720,6 +742,7 @@ def test_create_archive_with_option_true_calls_borg_without_corresponding_parame
 def test_create_archive_with_option_false_calls_borg_with_corresponding_parameter(option_name):
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -748,6 +771,7 @@ def test_create_archive_with_option_false_calls_borg_with_corresponding_paramete
 def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -776,6 +800,7 @@ def test_create_archive_with_files_cache_calls_borg_with_files_cache_parameters(
 def test_create_archive_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -804,6 +829,7 @@ def test_create_archive_with_local_path_calls_borg_via_local_path():
 def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -832,6 +858,7 @@ def test_create_archive_with_remote_path_calls_borg_with_remote_path_parameters(
 def test_create_archive_with_umask_calls_borg_with_umask_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -859,6 +886,7 @@ def test_create_archive_with_umask_calls_borg_with_umask_parameters():
 def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -886,6 +914,7 @@ def test_create_archive_with_lock_wait_calls_borg_with_lock_wait_parameters():
 def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_warning_output_log_level():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -914,6 +943,7 @@ def test_create_archive_with_stats_calls_borg_with_stats_parameter_and_warning_o
 def test_create_archive_with_stats_and_log_info_calls_borg_with_stats_parameter_and_info_output_log_level():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -943,6 +973,7 @@ def test_create_archive_with_stats_and_log_info_calls_borg_with_stats_parameter_
 def test_create_archive_with_files_calls_borg_with_list_parameter_and_warning_output_log_level():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -971,6 +1002,7 @@ def test_create_archive_with_files_calls_borg_with_list_parameter_and_warning_ou
 def test_create_archive_with_files_and_log_info_calls_borg_with_list_parameter_and_info_output_log_level():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -1000,6 +1032,7 @@ def test_create_archive_with_files_and_log_info_calls_borg_with_list_parameter_a
 def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_parameter_and_no_list():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -1029,6 +1062,7 @@ def test_create_archive_with_progress_and_log_info_calls_borg_with_progress_para
 def test_create_archive_with_progress_calls_borg_with_progress_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -1058,6 +1092,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr
     processes = flexmock()
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -1089,6 +1124,7 @@ def test_create_archive_with_progress_and_stream_processes_calls_borg_with_progr
 def test_create_archive_with_json_calls_borg_with_json_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -1119,6 +1155,7 @@ def test_create_archive_with_json_calls_borg_with_json_parameter():
 def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -1150,6 +1187,7 @@ def test_create_archive_with_stats_and_json_calls_borg_without_stats_parameter()
 def test_create_archive_with_source_directories_glob_expands():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'food'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -1178,6 +1216,7 @@ def test_create_archive_with_source_directories_glob_expands():
 def test_create_archive_with_non_matching_source_directories_glob_passes_through():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo*',))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -1206,6 +1245,7 @@ def test_create_archive_with_non_matching_source_directories_glob_passes_through
 def test_create_archive_with_glob_calls_borg_with_expanded_directories():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'food'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -1233,6 +1273,7 @@ def test_create_archive_with_glob_calls_borg_with_expanded_directories():
 def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -1260,6 +1301,7 @@ def test_create_archive_with_archive_name_format_calls_borg_with_archive_name():
 def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -1287,6 +1329,7 @@ def test_create_archive_with_archive_name_format_accepts_borg_placeholders():
 def test_create_archive_with_extra_borg_options_calls_borg_with_extra_options():
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -1315,6 +1358,7 @@ def test_create_archive_with_stream_processes_calls_borg_with_processes():
     processes = flexmock()
     flexmock(module).should_receive('borgmatic_source_directories').and_return([])
     flexmock(module).should_receive('deduplicate_directories').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('map_directories_to_devices').and_return({})
     flexmock(module).should_receive('_expand_directories').and_return(())
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)

tests/unit/borg/test_environment.py

@@ -62,13 +62,23 @@ def test_initialize_with_relocated_repo_access_should_override_default():
         os.environ = orig_environ
 
 
-def test_initialize_is_not_affected_by_existing_environment():
+def test_initialize_prefers_configuration_option_over_borg_environment_variable():
     orig_environ = os.environ
 
     try:
-        os.environ = {'BORG_PASSPHRASE': 'pass', 'BORG_SSH': 'mosh'}
+        os.environ = {'BORG_SSH': 'mosh'}
         module.initialize({'ssh_command': 'ssh -C'})
-        assert 'BORG_PASSPHRASE' not in os.environ
         assert os.environ.get('BORG_RSH') == 'ssh -C'
     finally:
         os.environ = orig_environ
+
+
+def test_initialize_passes_through_existing_borg_environment_variable():
+    orig_environ = os.environ
+
+    try:
+        os.environ = {'BORG_PASSPHRASE': 'pass'}
+        module.initialize({'ssh_command': 'ssh -C'})
+        assert os.environ.get('BORG_PASSPHRASE') == 'pass'
+    finally:
+        os.environ = orig_environ

tests/unit/hooks/test_postgresql.py

@@ -14,6 +14,7 @@ def test_dump_databases_runs_pg_dump_for_each_database():
         'databases/localhost/foo'
     ).and_return('databases/localhost/bar')
     flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+    flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
 
     for name, process in zip(('foo', 'bar'), processes):
         flexmock(module).should_receive('execute_command').with_args(
@@ -29,7 +30,7 @@ def test_dump_databases_runs_pg_dump_for_each_database():
                 'databases/localhost/{}'.format(name),
             ),
             shell=True,
-            extra_environment=None,
+            extra_environment={'PGSSLMODE': 'disable'},
             run_to_completion=False,
         ).and_return(process).once()
 
@@ -43,6 +44,7 @@ def test_dump_databases_with_dry_run_skips_pg_dump():
         'databases/localhost/foo'
     ).and_return('databases/localhost/bar')
     flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
+    flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
     flexmock(module).should_receive('execute_command').never()
 
     assert module.dump_databases(databases, 'test.yaml', {}, dry_run=True) == []
@@ -56,6 +58,7 @@ def test_dump_databases_runs_pg_dump_with_hostname_and_port():
         'databases/database.example.org/foo'
     )
     flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+    flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
 
     flexmock(module).should_receive('execute_command').with_args(
         (
@@ -74,7 +77,7 @@ def test_dump_databases_runs_pg_dump_with_hostname_and_port():
             'databases/database.example.org/foo',
         ),
         shell=True,
-        extra_environment=None,
+        extra_environment={'PGSSLMODE': 'disable'},
         run_to_completion=False,
     ).and_return(process).once()
 
@@ -89,6 +92,9 @@ def test_dump_databases_runs_pg_dump_with_username_and_password():
         'databases/localhost/foo'
     )
     flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+    flexmock(module).should_receive('make_extra_environment').and_return(
+        {'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}
+    )
 
     flexmock(module).should_receive('execute_command').with_args(
         (
@@ -105,13 +111,36 @@ def test_dump_databases_runs_pg_dump_with_username_and_password():
             'databases/localhost/foo',
         ),
         shell=True,
-        extra_environment={'PGPASSWORD': 'trustsome1'},
+        extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
         run_to_completion=False,
     ).and_return(process).once()
 
     assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
 
 
+def test_make_extra_environment_maps_options_to_environment():
+    database = {
+        'name': 'foo',
+        'password': 'pass',
+        'ssl_mode': 'require',
+        'ssl_cert': 'cert.crt',
+        'ssl_key': 'key.key',
+        'ssl_root_cert': 'root.crt',
+        'ssl_crl': 'crl.crl',
+    }
+    expected = {
+        'PGPASSWORD': 'pass',
+        'PGSSLMODE': 'require',
+        'PGSSLCERT': 'cert.crt',
+        'PGSSLKEY': 'key.key',
+        'PGSSLROOTCERT': 'root.crt',
+        'PGSSLCRL': 'crl.crl',
+    }
+
+    extra_env = module.make_extra_environment(database)
+    assert extra_env == expected
+
+
 def test_dump_databases_runs_pg_dump_with_directory_format():
     databases = [{'name': 'foo', 'format': 'directory'}]
     process = flexmock()
@@ -121,6 +150,7 @@ def test_dump_databases_runs_pg_dump_with_directory_format():
     )
     flexmock(module.dump).should_receive('create_parent_directory_for_dump')
     flexmock(module.dump).should_receive('create_named_pipe_for_dump').never()
+    flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
 
     flexmock(module).should_receive('execute_command').with_args(
         (
@@ -135,7 +165,7 @@ def test_dump_databases_runs_pg_dump_with_directory_format():
             'foo',
         ),
         shell=True,
-        extra_environment=None,
+        extra_environment={'PGSSLMODE': 'disable'},
         run_to_completion=False,
     ).and_return(process).once()
 
@@ -150,6 +180,7 @@ def test_dump_databases_runs_pg_dump_with_options():
         'databases/localhost/foo'
     )
     flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+    flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
 
     flexmock(module).should_receive('execute_command').with_args(
         (
@@ -165,7 +196,7 @@ def test_dump_databases_runs_pg_dump_with_options():
             'databases/localhost/foo',
         ),
         shell=True,
-        extra_environment=None,
+        extra_environment={'PGSSLMODE': 'disable'},
         run_to_completion=False,
     ).and_return(process).once()
 
@@ -180,11 +211,12 @@ def test_dump_databases_runs_pg_dumpall_for_all_databases():
         'databases/localhost/all'
     )
     flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+    flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
 
     flexmock(module).should_receive('execute_command').with_args(
         ('pg_dumpall', '--no-password', '--clean', '--if-exists', '>', 'databases/localhost/all'),
         shell=True,
-        extra_environment=None,
+        extra_environment={'PGSSLMODE': 'disable'},
         run_to_completion=False,
     ).and_return(process).once()
 
@@ -197,6 +229,7 @@ def test_restore_database_dump_runs_pg_restore():
 
     flexmock(module).should_receive('make_dump_path')
     flexmock(module.dump).should_receive('make_database_dump_filename')
+    flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
     flexmock(module).should_receive('execute_command_with_processes').with_args(
         (
             'pg_restore',
@@ -210,12 +243,12 @@ def test_restore_database_dump_runs_pg_restore():
         processes=[extract_process],
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
-        extra_environment=None,
+        extra_environment={'PGSSLMODE': 'disable'},
         borg_local_path='borg',
     ).once()
     flexmock(module).should_receive('execute_command').with_args(
         ('psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'),
-        extra_environment=None,
+        extra_environment={'PGSSLMODE': 'disable'},
     ).once()
 
     module.restore_database_dump(
@@ -228,6 +261,7 @@ def test_restore_database_dump_errors_on_multiple_database_config():
 
     flexmock(module).should_receive('make_dump_path')
     flexmock(module.dump).should_receive('make_database_dump_filename')
+    flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
     flexmock(module).should_receive('execute_command_with_processes').never()
     flexmock(module).should_receive('execute_command').never()
 
@@ -243,6 +277,7 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
 
     flexmock(module).should_receive('make_dump_path')
     flexmock(module.dump).should_receive('make_database_dump_filename')
+    flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
     flexmock(module).should_receive('execute_command_with_processes').with_args(
         (
             'pg_restore',
@@ -260,7 +295,7 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
         processes=[extract_process],
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
-        extra_environment=None,
+        extra_environment={'PGSSLMODE': 'disable'},
         borg_local_path='borg',
     ).once()
     flexmock(module).should_receive('execute_command').with_args(
@@ -277,7 +312,7 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
             '--command',
             'ANALYZE',
         ),
-        extra_environment=None,
+        extra_environment={'PGSSLMODE': 'disable'},
     ).once()
 
     module.restore_database_dump(
@@ -291,6 +326,9 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password():
 
     flexmock(module).should_receive('make_dump_path')
     flexmock(module.dump).should_receive('make_database_dump_filename')
+    flexmock(module).should_receive('make_extra_environment').and_return(
+        {'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'}
+    )
     flexmock(module).should_receive('execute_command_with_processes').with_args(
         (
             'pg_restore',
@@ -306,7 +344,7 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password():
         processes=[extract_process],
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
-        extra_environment={'PGPASSWORD': 'trustsome1'},
+        extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
         borg_local_path='borg',
     ).once()
     flexmock(module).should_receive('execute_command').with_args(
@@ -321,7 +359,7 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password():
             '--command',
             'ANALYZE',
         ),
-        extra_environment={'PGPASSWORD': 'trustsome1'},
+        extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
     ).once()
 
     module.restore_database_dump(
@@ -335,16 +373,18 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
 
     flexmock(module).should_receive('make_dump_path')
     flexmock(module.dump).should_receive('make_database_dump_filename')
+    flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
     flexmock(module).should_receive('execute_command_with_processes').with_args(
         ('psql', '--no-password'),
         processes=[extract_process],
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
-        extra_environment=None,
+        extra_environment={'PGSSLMODE': 'disable'},
         borg_local_path='borg',
     ).once()
     flexmock(module).should_receive('execute_command').with_args(
-        ('psql', '--no-password', '--quiet', '--command', 'ANALYZE'), extra_environment=None
+        ('psql', '--no-password', '--quiet', '--command', 'ANALYZE'),
+        extra_environment={'PGSSLMODE': 'disable'},
     ).once()
 
     module.restore_database_dump(
@@ -357,6 +397,7 @@ def test_restore_database_dump_with_dry_run_skips_restore():
 
     flexmock(module).should_receive('make_dump_path')
     flexmock(module.dump).should_receive('make_database_dump_filename')
+    flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
     flexmock(module).should_receive('execute_command_with_processes').never()
 
     module.restore_database_dump(
@@ -369,6 +410,7 @@ def test_restore_database_dump_without_extract_process_restores_from_disk():
 
     flexmock(module).should_receive('make_dump_path')
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return('/dump/path')
+    flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
     flexmock(module).should_receive('execute_command_with_processes').with_args(
         (
             'pg_restore',
@@ -383,12 +425,12 @@ def test_restore_database_dump_without_extract_process_restores_from_disk():
         processes=[],
         output_log_level=logging.DEBUG,
         input_file=None,
-        extra_environment=None,
+        extra_environment={'PGSSLMODE': 'disable'},
         borg_local_path='borg',
     ).once()
     flexmock(module).should_receive('execute_command').with_args(
         ('psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'),
-        extra_environment=None,
+        extra_environment={'PGSSLMODE': 'disable'},
     ).once()
 
     module.restore_database_dump(