Bump version for release.

Reviewed-on: https://projects.torsion.org/borgmatic-collective/borgmatic/pulls/907

NEWS

@@ -1,3 +1,20 @@
+1.8.14
+ * #896: Fix an error in borgmatic rcreate/init on an empty repository directory with Borg 1.4.
+ * #898: Add glob ("*") support to the "--repository" flag. Just quote any values containing
+   globs so your shell doesn't interpret them.
+ * #899: Fix for a "bad character" Borg error in which the "spot" check fed Borg an invalid pattern.
+ * #900: Fix for a potential traceback (TypeError) during the handling of another error.
+ * #904: Clarify the configuration reference about the "spot" check options:
+   https://torsion.org/borgmatic/docs/reference/configuration/
+ * #905: Fix the "source_directories_must_exist" option to work with relative "source_directories"
+   paths when a "working_directory" is set.
+ * #906: Add documentation details for how to run custom database dump commands using binaries from
+   running containers:
+   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#containers
+ * Fix a regression in which the "color" option had no effect.
+ * Add a recent contributors section to the documentation, because credit where credit's due! See:
+   https://torsion.org/borgmatic/#recent-contributors
+
 1.8.13
  * #298: Add "delete" and "rdelete" actions to delete archives or entire repositories.
  * #785: Add an "only_run_on" option to consistency checks so you can limit a check to running on

README.md

@@ -155,3 +155,7 @@ general, contributions are very welcome. We don't bite!
 Also, please check out the [borgmatic development
 how-to](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) for
 info on cloning source code, running tests, etc.
+
+### Recent contributors
+
+{% include borgmatic/contributors.html %}

borgmatic/actions/restore.py

@@ -147,7 +147,8 @@ def collect_archive_data_source_names(
         local_borg_version,
         global_arguments,
         list_paths=[
-            os.path.expanduser(
+            'sh:'
+            + os.path.expanduser(
                 borgmatic.hooks.dump.make_data_source_dump_path(borgmatic_source_directory, pattern)
             )
             for pattern in ('*_databases/*/*',)

borgmatic/borg/create.py

@@ -306,15 +306,25 @@ def collect_special_file_paths(
     )
 
 
-def check_all_source_directories_exist(source_directories):
+def check_all_source_directories_exist(source_directories, working_directory=None):
     '''
-    Given a sequence of source directories, check that they all exist. If any do not, raise an
-    exception.
+    Given a sequence of source directories and an optional working directory to serve as a prefix
+    for each (if it's a relative directory), check that the source directories all exist. If any do
+    not, raise an exception.
     '''
     missing_directories = [
         source_directory
         for source_directory in source_directories
-        if not all([os.path.exists(directory) for directory in expand_directory(source_directory)])
+        if not all(
+            [
+                os.path.exists(directory)
+                for directory in expand_directory(
+                    os.path.join(working_directory, source_directory)
+                    if working_directory
+                    else source_directory
+                )
+            ]
+        )
     ]
     if missing_directories:
         raise ValueError(f"Source directories do not exist: {', '.join(missing_directories)}")
@@ -342,8 +352,15 @@ def make_base_create_command(
     (base Borg create command flags, Borg create command positional arguments, open pattern file
     handle, open exclude file handle).
     '''
+    try:
+        working_directory = os.path.expanduser(config.get('working_directory'))
+    except TypeError:
+        working_directory = None
+
     if config.get('source_directories_must_exist', False):
-        check_all_source_directories_exist(config.get('source_directories'))
+        check_all_source_directories_exist(
+            config.get('source_directories'), working_directory=working_directory
+        )
 
     sources = deduplicate_directories(
         map_directories_to_devices(
@@ -445,11 +462,6 @@ def make_base_create_command(
         logger.warning(
             f'{repository_path}: Ignoring configured "read_special" value of false, as true is needed for database hooks.'
         )
-        try:
-            working_directory = os.path.expanduser(config.get('working_directory'))
-        except TypeError:
-            working_directory = None
-
         borg_environment = environment.make_environment(config)
 
         logger.debug(f'{repository_path}: Collecting special file paths')

borgmatic/borg/list.py

@@ -100,10 +100,11 @@ def capture_archive_listing(
     remote_path=None,
 ):
     '''
-    Given a local or remote repository path, an archive name, a configuration dict, the local Borg
-    version, global arguments as an argparse.Namespace, the archive paths in which to list files,
-    the Borg path format to use for the output, and local and remote Borg paths, capture the output
-    of listing that archive and return it as a list of file paths.
+    Given a local or remote repository path, an archive name, a configuration
+    dict, the local Borg version, global arguments as an argparse.Namespace,
+    the archive paths (or Borg patterns) in which to list files, the Borg path
+    format to use for the output, and local and remote Borg paths, capture the
+    output of listing that archive and return it as a list of file paths.
     '''
     borg_environment = environment.make_environment(config)
 
@@ -116,7 +117,7 @@ def capture_archive_listing(
                 argparse.Namespace(
                     repository=repository_path,
                     archive=archive,
-                    paths=[f'sh:{path}' for path in list_paths] if list_paths else None,
+                    paths=[path for path in list_paths] if list_paths else None,
                     find_paths=None,
                     json=None,
                     format=path_format or '{path}{NL}',  # noqa: FS003

borgmatic/borg/rcreate.py

@@ -9,7 +9,7 @@ from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 logger = logging.getLogger(__name__)
 
 
-RINFO_REPOSITORY_NOT_FOUND_EXIT_CODES = {2, 13}
+RINFO_REPOSITORY_NOT_FOUND_EXIT_CODES = {2, 13, 15}
 
 
 def create_repository(

borgmatic/commands/arguments.py

@@ -425,7 +425,7 @@ def make_parsers():
     )
     rcreate_group.add_argument(
         '--repository',
-        help='Path of the new repository to create (must be already specified in a borgmatic configuration file), defaults to the configured repository if there is only one',
+        help='Path of the new repository to create (must be already specified in a borgmatic configuration file), defaults to the configured repository if there is only one, quoted globs supported',
     )
     rcreate_group.add_argument(
         '--copy-crypt-key',
@@ -460,7 +460,7 @@ def make_parsers():
     transfer_group = transfer_parser.add_argument_group('transfer arguments')
     transfer_group.add_argument(
         '--repository',
-        help='Path of existing destination repository to transfer archives to, defaults to the configured repository if there is only one',
+        help='Path of existing destination repository to transfer archives to, defaults to the configured repository if there is only one, quoted globs supported',
     )
     transfer_group.add_argument(
         '--source-repository',
@@ -533,7 +533,7 @@ def make_parsers():
     prune_group = prune_parser.add_argument_group('prune arguments')
     prune_group.add_argument(
         '--repository',
-        help='Path of specific existing repository to prune (must be already specified in a borgmatic configuration file)',
+        help='Path of specific existing repository to prune (must be already specified in a borgmatic configuration file), quoted globs supported',
     )
     prune_group.add_argument(
         '--stats',
@@ -577,7 +577,7 @@ def make_parsers():
     compact_group = compact_parser.add_argument_group('compact arguments')
     compact_group.add_argument(
         '--repository',
-        help='Path of specific existing repository to compact (must be already specified in a borgmatic configuration file)',
+        help='Path of specific existing repository to compact (must be already specified in a borgmatic configuration file), quoted globs supported',
     )
     compact_group.add_argument(
         '--progress',
@@ -613,7 +613,7 @@ def make_parsers():
     create_group = create_parser.add_argument_group('create arguments')
     create_group.add_argument(
         '--repository',
-        help='Path of specific existing repository to backup to (must be already specified in a borgmatic configuration file)',
+        help='Path of specific existing repository to backup to (must be already specified in a borgmatic configuration file), quoted globs supported',
     )
     create_group.add_argument(
         '--progress',
@@ -647,7 +647,7 @@ def make_parsers():
     check_group = check_parser.add_argument_group('check arguments')
     check_group.add_argument(
         '--repository',
-        help='Path of specific existing repository to check (must be already specified in a borgmatic configuration file)',
+        help='Path of specific existing repository to check (must be already specified in a borgmatic configuration file), quoted globs supported',
     )
     check_group.add_argument(
         '--progress',
@@ -701,7 +701,7 @@ def make_parsers():
     delete_group = delete_parser.add_argument_group('delete arguments')
     delete_group.add_argument(
         '--repository',
-        help='Path of repository to delete or delete archives from, defaults to the configured repository if there is only one',
+        help='Path of repository to delete or delete archives from, defaults to the configured repository if there is only one, quoted globs supported',
     )
     delete_group.add_argument(
         '--archive',
@@ -792,7 +792,7 @@ def make_parsers():
     extract_group = extract_parser.add_argument_group('extract arguments')
     extract_group.add_argument(
         '--repository',
-        help='Path of repository to extract, defaults to the configured repository if there is only one',
+        help='Path of repository to extract, defaults to the configured repository if there is only one, quoted globs supported',
     )
     extract_group.add_argument(
         '--archive', help='Name of archive to extract (or "latest")', required=True
@@ -854,7 +854,7 @@ def make_parsers():
     )
     config_bootstrap_group.add_argument(
         '--repository',
-        help='Path of repository to extract config files from',
+        help='Path of repository to extract config files from, quoted globs supported',
         required=True,
     )
     config_bootstrap_group.add_argument(
@@ -952,7 +952,7 @@ def make_parsers():
     export_tar_group = export_tar_parser.add_argument_group('export-tar arguments')
     export_tar_group.add_argument(
         '--repository',
-        help='Path of repository to export from, defaults to the configured repository if there is only one',
+        help='Path of repository to export from, defaults to the configured repository if there is only one, quoted globs supported',
     )
     export_tar_group.add_argument(
         '--archive', help='Name of archive to export (or "latest")', required=True
@@ -998,7 +998,7 @@ def make_parsers():
     mount_group = mount_parser.add_argument_group('mount arguments')
     mount_group.add_argument(
         '--repository',
-        help='Path of repository to use, defaults to the configured repository if there is only one',
+        help='Path of repository to use, defaults to the configured repository if there is only one, quoted globs supported',
     )
     mount_group.add_argument('--archive', help='Name of archive to mount (or "latest")')
     mount_group.add_argument(
@@ -1080,7 +1080,7 @@ def make_parsers():
     rdelete_group = rdelete_parser.add_argument_group('delete arguments')
     rdelete_group.add_argument(
         '--repository',
-        help='Path of repository to delete, defaults to the configured repository if there is only one',
+        help='Path of repository to delete, defaults to the configured repository if there is only one, quoted globs supported',
     )
     rdelete_group.add_argument(
         '--list',
@@ -1117,7 +1117,7 @@ def make_parsers():
     restore_group = restore_parser.add_argument_group('restore arguments')
     restore_group.add_argument(
         '--repository',
-        help='Path of repository to restore from, defaults to the configured repository if there is only one',
+        help='Path of repository to restore from, defaults to the configured repository if there is only one, quoted globs supported',
     )
     restore_group.add_argument(
         '--archive', help='Name of archive to restore from (or "latest")', required=True
@@ -1171,7 +1171,7 @@ def make_parsers():
     rlist_group = rlist_parser.add_argument_group('rlist arguments')
     rlist_group.add_argument(
         '--repository',
-        help='Path of repository to list, defaults to the configured repositories',
+        help='Path of repository to list, defaults to the configured repositories, quoted globs supported',
     )
     rlist_group.add_argument(
         '--short', default=False, action='store_true', help='Output only archive names'
@@ -1231,7 +1231,7 @@ def make_parsers():
     list_group = list_parser.add_argument_group('list arguments')
     list_group.add_argument(
         '--repository',
-        help='Path of repository containing archive to list, defaults to the configured repositories',
+        help='Path of repository containing archive to list, defaults to the configured repositories, quoted globs supported',
     )
     list_group.add_argument('--archive', help='Name of the archive to list (or "latest")')
     list_group.add_argument(
@@ -1298,7 +1298,7 @@ def make_parsers():
     rinfo_group = rinfo_parser.add_argument_group('rinfo arguments')
     rinfo_group.add_argument(
         '--repository',
-        help='Path of repository to show info for, defaults to the configured repository if there is only one',
+        help='Path of repository to show info for, defaults to the configured repository if there is only one, quoted globs supported',
     )
     rinfo_group.add_argument(
         '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
@@ -1315,7 +1315,7 @@ def make_parsers():
     info_group = info_parser.add_argument_group('info arguments')
     info_group.add_argument(
         '--repository',
-        help='Path of repository containing archive to show info for, defaults to the configured repository if there is only one',
+        help='Path of repository containing archive to show info for, defaults to the configured repository if there is only one, quoted globs supported',
     )
     info_group.add_argument('--archive', help='Name of archive to show info for (or "latest")')
     info_group.add_argument(
@@ -1376,7 +1376,7 @@ def make_parsers():
     break_lock_group = break_lock_parser.add_argument_group('break-lock arguments')
     break_lock_group.add_argument(
         '--repository',
-        help='Path of repository to break the lock for, defaults to the configured repository if there is only one',
+        help='Path of repository to break the lock for, defaults to the configured repository if there is only one, quoted globs supported',
     )
     break_lock_group.add_argument(
         '-h', '--help', action='help', help='Show this help message and exit'
@@ -1416,7 +1416,7 @@ def make_parsers():
     )
     key_export_group.add_argument(
         '--repository',
-        help='Path of repository to export the key for, defaults to the configured repository if there is only one',
+        help='Path of repository to export the key for, defaults to the configured repository if there is only one, quoted globs supported',
     )
     key_export_group.add_argument(
         '--path',
@@ -1437,7 +1437,7 @@ def make_parsers():
     borg_group = borg_parser.add_argument_group('borg arguments')
     borg_group.add_argument(
         '--repository',
-        help='Path of repository to pass to Borg, defaults to the configured repositories',
+        help='Path of repository to pass to Borg, defaults to the configured repositories, quoted globs supported',
     )
     borg_group.add_argument('--archive', help='Name of archive to pass to Borg (or "latest")')
     borg_group.add_argument(

borgmatic/commands/borgmatic.py

@@ -579,7 +579,7 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True):
                         )
                     ),
                     logging.makeLogRecord(
-                        dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
+                        dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=str(error))
                     ),
                 ]
             )
@@ -614,13 +614,13 @@ def log_error_records(
     level_name = logging._levelToName[levelno]
 
     if not error:
-        yield log_record(levelno=levelno, levelname=level_name, msg=message)
+        yield log_record(levelno=levelno, levelname=level_name, msg=str(message))
         return
 
     try:
         raise error
     except CalledProcessError as error:
-        yield log_record(levelno=levelno, levelname=level_name, msg=message)
+        yield log_record(levelno=levelno, levelname=level_name, msg=str(message))
 
         if error.output:
             try:
@@ -639,10 +639,10 @@ def log_error_records(
                     suppress_log=True,
                 )
 
-        yield log_record(levelno=levelno, levelname=level_name, msg=error)
+        yield log_record(levelno=levelno, levelname=level_name, msg=str(error))
     except (ValueError, OSError) as error:
-        yield log_record(levelno=levelno, levelname=level_name, msg=message)
-        yield log_record(levelno=levelno, levelname=level_name, msg=error)
+        yield log_record(levelno=levelno, levelname=level_name, msg=str(message))
+        yield log_record(levelno=levelno, levelname=level_name, msg=str(error))
     except:  # noqa: E722
         # Raising above only as a means of determining the error type. Swallow the exception here
         # because we don't want the exception to propagate out of this function.

borgmatic/config/schema.yaml

@@ -521,18 +521,20 @@ properties:
                               - extract
                               - disabled
                           description: |
-                              Name of consistency check to run: "repository",
-                              "archives", "data", "spot", and/or "extract".
-                              "repository" checks the consistency of the
-                              repository, "archives" checks all of the
-                              archives, "data" verifies the integrity of the
-                              data within the archives, "spot" checks that
-                              some percentage of source files are found in the
-                              most recent archive (with identical contents),
-                              and "extract" does an extraction dry-run of the
-                              most recent archive. Note that "data" implies
-                              "archives". See "skip_actions" for disabling
-                              checks altogether.
+                              Name of the consistency check to run:
+                               * "repository" checks the consistency of the
+                              repository.
+                               * "archives" checks all of the archives.
+                               * "data" verifies the integrity of the data
+                              within the archives and implies the "archives"
+                              check as well.
+                               * "spot" checks that some percentage of source
+                              files are found in the most recent archive (with
+                              identical contents).
+                               * "extract" does an extraction dry-run of the
+                              most recent archive.
+                               * See "skip_actions" for disabling checks
+                              altogether.
                           example: spot
                       frequency:
                           type: string
@@ -568,18 +570,20 @@ properties:
                           enum:
                               - repository
                           description: |
-                              Name of consistency check to run: "repository",
-                              "archives", "data", "spot", and/or "extract".
-                              "repository" checks the consistency of the
-                              repository, "archives" checks all of the
-                              archives, "data" verifies the integrity of the
-                              data within the archives, "spot" checks that
-                              some percentage of source files are found in the
-                              most recent archive (with identical contents),
-                              and "extract" does an extraction dry-run of the
-                              most recent archive. Note that "data" implies
-                              "archives". See "skip_actions" for disabling
-                              checks altogether.
+                              Name of the consistency check to run:
+                               * "repository" checks the consistency of the
+                              repository.
+                               * "archives" checks all of the archives.
+                               * "data" verifies the integrity of the data
+                              within the archives and implies the "archives"
+                              check as well.
+                               * "spot" checks that some percentage of source
+                              files are found in the most recent archive (with
+                              identical contents).
+                               * "extract" does an extraction dry-run of the
+                              most recent archive.
+                               * See "skip_actions" for disabling checks
+                              altogether.
                           example: spot
                       frequency:
                           type: string
@@ -630,18 +634,20 @@ properties:
                           enum:
                               - spot
                           description: |
-                              Name of consistency check to run: "repository",
-                              "archives", "data", "spot", and/or "extract".
-                              "repository" checks the consistency of the
-                              repository, "archives" checks all of the
-                              archives, "data" verifies the integrity of the
-                              data within the archives, "spot" checks that
-                              some percentage of source files are found in the
-                              most recent archive (with identical contents),
-                              and "extract" does an extraction dry-run of the
-                              most recent archive. Note that "data" implies
-                              "archives". See "skip_actions" for disabling
-                              checks altogether.
+                              Name of the consistency check to run:
+                               * "repository" checks the consistency of the
+                              repository.
+                               * "archives" checks all of the archives.
+                               * "data" verifies the integrity of the data
+                              within the archives and implies the "archives"
+                              check as well.
+                               * "spot" checks that some percentage of source
+                              files are found in the most recent archive (with
+                              identical contents).
+                               * "extract" does an extraction dry-run of the
+                              most recent archive.
+                               * See "skip_actions" for disabling checks
+                              altogether.
                           example: repository
                       frequency:
                           type: string
@@ -677,7 +683,8 @@ properties:
                               archive file count that is allowed before the
                               entire consistency check fails. This can catch
                               problems like incorrect excludes, inadvertent
-                              deletes, etc. Only applies to the "spot" check.
+                              deletes, etc. Required (and only valid) for the
+                              "spot" check.
                           example: 10
                       data_sample_percentage:
                           type: number
@@ -685,7 +692,8 @@ properties:
                               The percentage of total files in the source
                               directories to randomly sample and compare to
                               their corresponding files in the most recent
-                              backup archive. Only applies to the "spot" check.
+                              backup archive. Required (and only valid) for the
+                              "spot" check.
                           example: 1
                       data_tolerance_percentage:
                           type: number
@@ -697,7 +705,7 @@ properties:
                               been bulk-changed by malware, backups that have
                               been tampered with, etc. The value must be lower
                               than or equal to the "contents_sample_percentage".
-                              Only applies to the "spot" check.
+                              Required (and only valid) for the "spot" check.
                           example: 0.5
                       xxh64sum_command:
                           type: string
@@ -706,7 +714,7 @@ properties:
                               source files, usually found in an OS package named
                               "xxhash". Do not substitute with a different hash
                               type (SHA, MD5, etc.) or the check will never
-                              succeed. Only applies to the "spot" check.
+                              succeed. Only valid for the "spot" check.
                           example: /usr/local/bin/xxh64sum
         description: |
             List of one or more consistency checks to run on a periodic basis
@@ -1036,9 +1044,12 @@ properties:
                     description: |
                         Command to use instead of "pg_dump" or "pg_dumpall".
                         This can be used to run a specific pg_dump version
-                        (e.g., one inside a running container). Defaults to
-                        "pg_dump" for single database dump or "pg_dumpall" to
-                        dump all databases.
+                        (e.g., one inside a running container). If you run it
+                        from within a container, make sure to mount your
+                        host's ".borgmatic" folder into the container using
+                        the same directory structure. Defaults to "pg_dump"
+                        for single database dump or "pg_dumpall" to dump all
+                        databases.
                     example: docker exec my_pg_container pg_dump
                 pg_restore_command:
                     type: string
@@ -1152,7 +1163,10 @@ properties:
                     description: |
                         Command to use instead of "mariadb-dump". This can be
                         used to run a specific mariadb_dump version (e.g., one
-                        inside a running container). Defaults to "mariadb-dump".
+                        inside a running container). If you run it from within
+                        a container, make sure to mount your host's
+                        ".borgmatic" folder into the container using the same
+                        directory structure. Defaults to "mariadb-dump".
                     example: docker exec mariadb_container mariadb-dump
                 mariadb_command:
                     type: string
@@ -1280,9 +1294,12 @@ properties:
                 mysql_dump_command:
                     type: string
                     description: |
-                        Command to use instead of "mysqldump". This can be used
-                        to run a specific mysql_dump version (e.g., one inside a
-                        running container). Defaults to "mysqldump".
+                        Command to use instead of "mysqldump". This can be
+                        used to run a specific mysql_dump version (e.g., one
+                        inside a running container). If you run it from within
+                        a container, make sure to mount your host's
+                        ".borgmatic" folder into the container using the same
+                        directory structure. Defaults to "mysqldump".
                     example: docker exec mysql_container mysqldump
                 mysql_command:
                     type: string

borgmatic/config/validate.py

@@ -1,3 +1,4 @@
+import fnmatch
 import os
 
 import jsonschema
@@ -149,18 +150,30 @@ def normalize_repository_path(repository):
         return repository
 
 
+def glob_match(first, second):
+    '''
+    Given two strings, return whether the first matches the second. Globs are
+    supported.
+    '''
+    if first is None or second is None:
+        return False
+
+    return fnmatch.fnmatch(first, second) or fnmatch.fnmatch(second, first)
+
+
 def repositories_match(first, second):
     '''
-    Given two repository dicts with keys 'path' (relative and/or absolute),
-    and 'label', or two repository paths, return whether they match.
+    Given two repository dicts with keys "path" (relative and/or absolute),
+    and "label", two repository paths as strings, or a mix of the two formats,
+    return whether they match. Globs are supported.
     '''
     if isinstance(first, str):
         first = {'path': first, 'label': first}
     if isinstance(second, str):
         second = {'path': second, 'label': second}
-    return (first.get('label') == second.get('label')) or (
-        normalize_repository_path(first.get('path'))
-        == normalize_repository_path(second.get('path'))
+
+    return glob_match(first.get('label'), second.get('label')) or glob_match(
+        normalize_repository_path(first.get('path')), normalize_repository_path(second.get('path'))
     )
 
 

borgmatic/logger.py

@@ -38,7 +38,7 @@ def should_do_markup(no_color, configs):
     if no_color:
         return False
 
-    if any(config.get('output', {}).get('color') is False for config in configs.values()):
+    if any(config.get('color', True) is False for config in configs.values()):
         return False
 
     if os.environ.get('NO_COLOR', None):

docs/Dockerfile

@@ -1,14 +1,15 @@
-FROM docker.io/alpine:3.17.1 as borgmatic
+FROM docker.io/alpine:3.20.1 AS borgmatic
 
 COPY . /app
 RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib
-RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
+RUN pip install --break-system-packages --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
 RUN borgmatic --help > /command-line.txt \
     && for action in rcreate transfer create prune compact check delete extract config "config bootstrap" "config generate" "config validate" export-tar mount umount rdelete restore rlist list rinfo info break-lock borg; do \
            echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \
            && borgmatic $action --help >> /command-line.txt; done
+RUN /app/docs/fetch-contributors >> /contributors.html
 
-FROM docker.io/node:19.5.0-alpine as html
+FROM docker.io/node:22.4.0-alpine AS html
 
 ARG ENVIRONMENT=production
 
@@ -24,11 +25,12 @@ RUN npm install @11ty/eleventy \
     markdown-it-replace-link
 COPY --from=borgmatic /etc/borgmatic/config.yaml /source/docs/_includes/borgmatic/config.yaml
 COPY --from=borgmatic /command-line.txt /source/docs/_includes/borgmatic/command-line.txt
+COPY --from=borgmatic /contributors.html /source/docs/_includes/borgmatic/contributors.html
 COPY . /source
 RUN NODE_ENV=${ENVIRONMENT} npx eleventy --input=/source/docs --output=/output/docs \
   && mv /output/docs/index.html /output/index.html
 
-FROM docker.io/nginx:1.22.1-alpine
+FROM docker.io/nginx:1.26.1-alpine
 
 COPY --from=html /output /usr/share/nginx/html
 COPY --from=borgmatic /etc/borgmatic/config.yaml /usr/share/nginx/html/docs/reference/config.yaml

docs/fetch-contributors

@@ -0,0 +1,68 @@
+#!/usr/bin/python
+
+'''
+A script to fetch recent contributors to borgmatic, used during documentation generation.
+'''
+
+import datetime
+import itertools
+import operator
+import subprocess
+
+import requests
+
+
+def list_merged_pulls(url):
+    '''
+    Given a Gitea or GitHub API endpoint URL for pull requests, fetch and return the corresponding
+    JSON for all such merged pull requests.
+    '''
+    response = requests.get(f'{url}?state=closed', headers={'Accept': 'application/json', 'Content-Type': 'application/json'})
+
+    if not response.ok:
+        response.raise_for_status()
+
+    return tuple(pull for pull in response.json() if pull.get('merged_at'))
+
+
+API_ENDPOINT_URLS = (
+    'https://projects.torsion.org/api/v1/repos/borgmatic-collective/borgmatic/pulls',
+    'https://api.github.com/repos/borgmatic-collective/borgmatic/pulls',
+)
+RECENT_CONTRIBUTORS_CUTOFF_DAYS = 365
+
+
+def print_contributors():
+    '''
+    Display the recent contributors as a row of avatars in an HTML fragment.
+    '''
+    pulls = tuple(itertools.chain.from_iterable(list_merged_pulls(url) for url in API_ENDPOINT_URLS))
+    seen_user_ids = set()
+
+    print('<p>')
+
+    for pull in sorted(pulls, key=operator.itemgetter('merged_at'), reverse=True):
+        merged_at = pull.get('merged_at')
+        user = pull.get('user')
+
+        if not merged_at or not user:
+            continue
+
+        user_id = user.get('id')
+
+        if not user_id or user_id in seen_user_ids:
+            continue
+
+        if datetime.datetime.fromisoformat(merged_at) < datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=RECENT_CONTRIBUTORS_CUTOFF_DAYS):
+            continue
+
+        seen_user_ids.add(user_id)
+        print(
+            f'''<a href="{user.get('html_url')}?tab=activity"><img src="{user.get('avatar_url')}" width="50" height="50" title="{user.get('full_name') or user.get('login')}" /></a>'''
+        )
+
+    print('</p>')
+
+
+if __name__ == '__main__':
+    print_contributors()

docs/how-to/backup-your-databases.md

@@ -229,7 +229,20 @@ hooks:
 
 ... where `my_pg_container` is the name of your database container. In this
 example, you'd also need to set the `pg_restore_command` and `psql_command`
-options.
+options. If you choose to use the `pg_dump` command within the container
+though, note that it will output the database dump to a file inside the
+container. So you'll have to mount the `.borgmatic` folder from your host's
+home folder into the container using the same directory structure.
+
+See the following Docker compose file an as example:
+
+```yaml
+services:
+  db:
+    image: postgres
+    volumes:
+      - /home/USERNAME/.borgmatic:/home/USERNAME/.borgmatic
+```
 
 Similar command override options are available for (some of) the other
 supported database types as well. See the [configuration

docs/how-to/customize-warnings-and-errors.md

@@ -48,7 +48,7 @@ support for [more granular exit
 codes](https://borgbackup.readthedocs.io/en/1.4-maint/usage/general.html#return-codes)
 means that you can configure borgmatic to respond to specific Borg conditions.
 See the full list of [Borg 1.4 error and warning exit
-codes](https://borgbackup.readthedocs.io/en/1.4.0b2/internals/frontends.html#message-ids).
+codes](https://borgbackup.readthedocs.io/en/stable/internals/frontends.html#message-ids).
 The `rc:` numeric value there tells you the exit code for each.
 
 For instance, this borgmatic configuration elevates all Borg backup file

docs/how-to/develop-on-borgmatic.md

@@ -157,8 +157,12 @@ the following deviations from it:
  * Import fully qualified Python modules instead of importing individual
    functions, classes, or constants. E.g., do `import os.path` instead of
    `from os import path`. (Some exceptions to this are made in tests.)
+ * Only use classes and OOP as a last resort, such as when integrating with
+   Python libraries that require it.
+ * Prefer functional code where it makes sense, e.g. when constructing a
+   command (to subsequently execute imperatively).
 
-borgmatic code uses the [Black](https://black.readthedocs.io/en/stable/) code
+borgmatic uses the [Black](https://black.readthedocs.io/en/stable/) code
 formatter, the [Flake8](http://flake8.pycqa.org/en/latest/) code checker, and
 the [isort](https://github.com/timothycrosley/isort) import orderer, so
 certain code style requirements are enforced when running automated tests. See

scripts/run-full-tests

@@ -21,7 +21,7 @@ apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client m
     py3-ruamel.yaml py3-ruamel.yaml.clib py3-yaml bash sqlite fish
 # If certain dependencies of black are available in this version of Alpine, install them.
 apk add --no-cache py3-typed-ast py3-regex || true
-python3 -m pip install --no-cache --upgrade pip==22.2.2 setuptools==64.0.1
+python3 -m pip install --no-cache --upgrade pip==24.2 setuptools==72.1.0
 pip3 install --ignore-installed tox==4.11.3
 export COVERAGE_FILE=/tmp/.coverage
 

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.8.13'
+VERSION = '1.8.14'
 
 
 setup(

test_requirements.txt

@@ -2,7 +2,7 @@ appdirs==1.4.4
 apprise==1.8.0
 attrs==23.2.0
 black==24.4.2
-certifi==2024.2.2
+certifi==2024.7.4
 chardet==5.2.0
 click==8.1.7
 codespell==2.2.6

tests/end-to-end/test_validate_config.py

@@ -9,7 +9,7 @@ def test_validate_config_command_with_valid_configuration_succeeds():
         config_path = os.path.join(temporary_directory, 'test.yaml')
 
         subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' '))
-        exit_code = subprocess.call(f'validate-borgmatic-config --config {config_path}'.split(' '))
+        exit_code = subprocess.call(f'borgmatic config validate --config {config_path}'.split(' '))
 
         assert exit_code == 0
 
@@ -24,7 +24,7 @@ def test_validate_config_command_with_invalid_configuration_fails():
         config_file.write(config)
         config_file.close()
 
-        exit_code = subprocess.call(f'validate-borgmatic-config --config {config_path}'.split(' '))
+        exit_code = subprocess.call(f'borgmatic config validate --config {config_path}'.split(' '))
 
         assert exit_code == 1
 
@@ -35,7 +35,7 @@ def test_validate_config_command_with_show_flag_displays_configuration():
 
         subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' '))
         output = subprocess.check_output(
-            f'validate-borgmatic-config --config {config_path} --show'.split(' ')
+            f'borgmatic config validate --config {config_path} --show'.split(' ')
         ).decode(sys.stdout.encoding)
 
         assert 'repositories:' in output

tests/unit/borg/test_create.py

@@ -1891,3 +1891,18 @@ def test_check_all_source_directories_exist_with_non_existent_directory_raises()
 
     with pytest.raises(ValueError):
         module.check_all_source_directories_exist(['foo'])
+
+
+def test_check_all_source_directories_exist_with_working_directory_applies_to_relative_source_directories():
+    flexmock(module).should_receive('expand_directory').with_args('/tmp/foo*').and_return(
+        ('/tmp/foo', '/tmp/food')
+    )
+    flexmock(module).should_receive('expand_directory').with_args('/root/bar').and_return(
+        ('/root/bar',)
+    )
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.os.path).should_receive('exists').with_args('/tmp/foo').and_return(True)
+    flexmock(module.os.path).should_receive('exists').with_args('/tmp/food').and_return(True)
+    flexmock(module.os.path).should_receive('exists').with_args('/root/bar').and_return(True)
+
+    module.check_all_source_directories_exist(['foo*', '/root/bar'], working_directory='/tmp')

tests/unit/config/test_validate.py

@@ -62,7 +62,7 @@ def test_validation_error_string_contains_errors():
 
 
 def test_apply_logical_validation_raises_if_unknown_repository_in_check_repositories():
-    flexmock(module).format_json_error = lambda error: error.message
+    flexmock(module).should_receive('repositories_match').and_return(False)
 
     with pytest.raises(module.Validation_error):
         module.apply_logical_validation(
@@ -75,7 +75,9 @@ def test_apply_logical_validation_raises_if_unknown_repository_in_check_reposito
         )
 
 
-def test_apply_logical_validation_does_not_raise_if_known_repository_path_in_check_repositories():
+def test_apply_logical_validation_does_not_raise_if_known_repository_in_check_repositories():
+    flexmock(module).should_receive('repositories_match').and_return(True)
+
     module.apply_logical_validation(
         'config.yaml',
         {
@@ -86,35 +88,6 @@ def test_apply_logical_validation_does_not_raise_if_known_repository_path_in_che
     )
 
 
-def test_apply_logical_validation_does_not_raise_if_known_repository_label_in_check_repositories():
-    module.apply_logical_validation(
-        'config.yaml',
-        {
-            'repositories': [
-                {'path': 'repo.borg', 'label': 'my_repo'},
-                {'path': 'other.borg', 'label': 'other_repo'},
-            ],
-            'keep_secondly': 1000,
-            'check_repositories': ['my_repo'],
-        },
-    )
-
-
-def test_apply_logical_validation_does_not_raise_if_archive_name_format_and_prefix_present():
-    module.apply_logical_validation(
-        'config.yaml',
-        {
-            'archive_name_format': '{hostname}-{now}',  # noqa: FS003
-            'prefix': '{hostname}-',  # noqa: FS003
-            'prefix': '{hostname}-',  # noqa: FS003
-        },
-    )
-
-
-def test_apply_logical_validation_does_not_raise_otherwise():
-    module.apply_logical_validation('config.yaml', {'keep_secondly': 1000})
-
-
 def test_normalize_repository_path_passes_through_remote_repository():
     repository = 'example.org:test.borg'
 
@@ -143,39 +116,94 @@ def test_normalize_repository_path_resolves_relative_repository():
     module.normalize_repository_path(repository) == absolute
 
 
-def test_repositories_match_does_not_raise():
+@pytest.mark.parametrize(
+    'first,second,expected_result',
+    (
+        (None, None, False),
+        ('foo', None, False),
+        (None, 'bar', False),
+        ('foo', 'foo', True),
+        ('foo', 'bar', False),
+        ('foo*', 'foof', True),
+        ('barf', 'bar*', True),
+        ('foo*', 'bar*', False),
+    ),
+)
+def test_glob_match_matches_globs(first, second, expected_result):
+    assert module.glob_match(first=first, second=second) is expected_result
+
+
+def test_repositories_match_matches_on_path():
     flexmock(module).should_receive('normalize_repository_path')
-
-    module.repositories_match('foo', 'bar')
-
-
-def test_guard_configuration_contains_repository_does_not_raise_when_repository_in_config():
-    flexmock(module).should_receive('repositories_match').replace_with(
+    flexmock(module).should_receive('glob_match').replace_with(
         lambda first, second: first == second
     )
 
-    module.guard_configuration_contains_repository(
-        repository='repo', configurations={'config.yaml': {'repositories': ['repo']}}
+    module.repositories_match(
+        {'path': 'foo', 'label': 'my repo'}, {'path': 'foo', 'label': 'other repo'}
+    ) is True
+
+
+def test_repositories_match_matches_on_label():
+    flexmock(module).should_receive('normalize_repository_path')
+    flexmock(module).should_receive('glob_match').replace_with(
+        lambda first, second: first == second
     )
 
+    module.repositories_match(
+        {'path': 'foo', 'label': 'my repo'}, {'path': 'bar', 'label': 'my repo'}
+    ) is True
+
+
+def test_repositories_match_with_different_paths_and_labels_does_not_match():
+    flexmock(module).should_receive('normalize_repository_path')
+    flexmock(module).should_receive('glob_match').replace_with(
+        lambda first, second: first == second
+    )
+
+    module.repositories_match(
+        {'path': 'foo', 'label': 'my repo'}, {'path': 'bar', 'label': 'other repo'}
+    ) is False
+
+
+def test_repositories_match_matches_on_string_repository():
+    flexmock(module).should_receive('normalize_repository_path')
+    flexmock(module).should_receive('glob_match').replace_with(
+        lambda first, second: first == second
+    )
+
+    module.repositories_match('foo', 'foo') is True
+
+
+def test_repositories_match_with_different_string_repositories_does_not_match():
+    flexmock(module).should_receive('normalize_repository_path')
+    flexmock(module).should_receive('glob_match').replace_with(
+        lambda first, second: first == second
+    )
+
+    module.repositories_match('foo', 'bar') is False
+
+
+def test_repositories_match_supports_mixed_repositories():
+    flexmock(module).should_receive('normalize_repository_path')
+    flexmock(module).should_receive('glob_match').replace_with(
+        lambda first, second: first == second
+    )
+
+    module.repositories_match({'path': 'foo', 'label': 'my foo'}, 'bar') is False
+
+
+def test_guard_configuration_contains_repository_does_not_raise_when_repository_matches():
+    flexmock(module).should_receive('repositories_match').and_return(True)
 
-def test_guard_configuration_contains_repository_does_not_raise_when_repository_label_in_config():
     module.guard_configuration_contains_repository(
         repository='repo',
         configurations={'config.yaml': {'repositories': [{'path': 'foo/bar', 'label': 'repo'}]}},
     )
 
 
-def test_guard_configuration_contains_repository_does_not_raise_when_repository_not_given():
-    module.guard_configuration_contains_repository(
-        repository=None, configurations={'config.yaml': {'repositories': ['repo']}}
-    )
-
-
-def test_guard_configuration_contains_repository_errors_when_repository_missing_from_config():
-    flexmock(module).should_receive('repositories_match').replace_with(
-        lambda first, second: first == second
-    )
+def test_guard_configuration_contains_repository_errors_when_repository_does_not_match():
+    flexmock(module).should_receive('repositories_match').and_return(False)
 
     with pytest.raises(ValueError):
         module.guard_configuration_contains_repository(

tests/unit/test_logger.py

@@ -45,23 +45,27 @@ def test_interactive_console_true_when_isatty_and_TERM_is_not_dumb(capsys):
 
 
 def test_should_do_markup_respects_no_color_value():
+    flexmock(module).should_receive('interactive_console').never()
     assert module.should_do_markup(no_color=True, configs={}) is False
 
 
 def test_should_do_markup_respects_config_value():
-    assert (
-        module.should_do_markup(no_color=False, configs={'foo.yaml': {'output': {'color': False}}})
-        is False
-    )
+    flexmock(module).should_receive('interactive_console').never()
+    assert module.should_do_markup(no_color=False, configs={'foo.yaml': {'color': False}}) is False
+
+    flexmock(module).should_receive('interactive_console').and_return(True).once()
+    assert module.should_do_markup(no_color=False, configs={'foo.yaml': {'color': True}}) is True
 
 
 def test_should_do_markup_prefers_any_false_config_value():
+    flexmock(module).should_receive('interactive_console').never()
+
     assert (
         module.should_do_markup(
             no_color=False,
             configs={
-                'foo.yaml': {'output': {'color': True}},
-                'bar.yaml': {'output': {'color': False}},
+                'foo.yaml': {'color': True},
+                'bar.yaml': {'color': False},
             },
         )
         is False
@@ -80,25 +84,23 @@ def test_should_do_markup_respects_PY_COLORS_environment_variable():
 
 
 def test_should_do_markup_prefers_no_color_value_to_config_value():
-    assert (
-        module.should_do_markup(no_color=True, configs={'foo.yaml': {'output': {'color': True}}})
-        is False
-    )
+    flexmock(module).should_receive('interactive_console').never()
+
+    assert module.should_do_markup(no_color=True, configs={'foo.yaml': {'color': True}}) is False
 
 
 def test_should_do_markup_prefers_config_value_to_environment_variables():
     flexmock(module.os.environ).should_receive('get').and_return('True')
     flexmock(module).should_receive('to_bool').and_return(True)
+    flexmock(module).should_receive('interactive_console').never()
 
-    assert (
-        module.should_do_markup(no_color=False, configs={'foo.yaml': {'output': {'color': False}}})
-        is False
-    )
+    assert module.should_do_markup(no_color=False, configs={'foo.yaml': {'color': False}}) is False
 
 
 def test_should_do_markup_prefers_no_color_value_to_environment_variables():
     flexmock(module.os.environ).should_receive('get').and_return('True')
     flexmock(module).should_receive('to_bool').and_return(True)
+    flexmock(module).should_receive('interactive_console').never()
 
     assert module.should_do_markup(no_color=True, configs={}) is False
 
@@ -124,6 +126,7 @@ def test_should_do_markup_prefers_PY_COLORS_to_interactive_console_value():
 def test_should_do_markup_prefers_NO_COLOR_to_interactive_console_value():
     flexmock(module.os.environ).should_receive('get').with_args('PY_COLORS', None).and_return(None)
     flexmock(module.os.environ).should_receive('get').with_args('NO_COLOR', None).and_return('True')
+    flexmock(module).should_receive('interactive_console').never()
 
     assert module.should_do_markup(no_color=False, configs={}) is False
 
@@ -131,6 +134,7 @@ def test_should_do_markup_prefers_NO_COLOR_to_interactive_console_value():
 def test_should_do_markup_respects_NO_COLOR_environment_variable():
     flexmock(module.os.environ).should_receive('get').with_args('NO_COLOR', None).and_return('True')
     flexmock(module.os.environ).should_receive('get').with_args('PY_COLORS', None).and_return(None)
+    flexmock(module).should_receive('interactive_console').never()
 
     assert module.should_do_markup(no_color=False, configs={}) is False
 
@@ -150,6 +154,7 @@ def test_should_do_markup_prefers_NO_COLOR_to_PY_COLORS():
     flexmock(module.os.environ).should_receive('get').with_args('NO_COLOR', None).and_return(
         'SomeValue'
     )
+    flexmock(module).should_receive('interactive_console').never()
 
     assert module.should_do_markup(no_color=False, configs={}) is False