Bump version for release.

Merge pull request #73 from diivi/feat/restore-with-different-hostname-port-username

.drone.yml

@@ -1,29 +1,52 @@
+---
 kind: pipeline
 name: python-3-8-alpine-3-13
 
 services:
   - name: postgresql
-    image: postgres:13.1-alpine
+    image: docker.io/postgres:13.1-alpine
     environment:
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
+  - name: postgresql2
+    image: docker.io/postgres:13.1-alpine
+    environment:
+      POSTGRES_PASSWORD: test2
+      POSTGRES_DB: test
+      POSTGRES_USER: postgres2
+    commands:
+      - docker-entrypoint.sh -p 5433
   - name: mysql
-    image: mariadb:10.5
+    image: docker.io/mariadb:10.5
     environment:
       MYSQL_ROOT_PASSWORD: test
       MYSQL_DATABASE: test
+  - name: mysql2
+    image: docker.io/mariadb:10.5
+    environment:
+      MYSQL_ROOT_PASSWORD: test2
+      MYSQL_DATABASE: test
+    commands:
+      - docker-entrypoint.sh --port=3307
   - name: mongodb
-    image: mongo:5.0.5
+    image: docker.io/mongo:5.0.5
     environment:
       MONGO_INITDB_ROOT_USERNAME: root
       MONGO_INITDB_ROOT_PASSWORD: test
+  - name: mongodb2
+    image: docker.io/mongo:5.0.5
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: root2
+      MONGO_INITDB_ROOT_PASSWORD: test2
+    commands:
+      - docker-entrypoint.sh --port=27018
 
 clone:
   skip_verify: true
 
 steps:
 - name: build
-  image: alpine:3.13
+  image: docker.io/alpine:3.13
   environment:
     TEST_CONTAINER: true
   pull: always
@@ -32,27 +55,32 @@ steps:
 ---
 kind: pipeline
 name: documentation
+type: exec
+
+platform:
+  os: linux
+  arch: amd64
 
 clone:
   skip_verify: true
 
 steps:
 - name: build
-  image: plugins/docker
-  settings:
-    username:
+  environment:
+    USERNAME:
       from_secret: docker_username
-    password:
+    PASSWORD:
       from_secret: docker_password
-    registry: projects.torsion.org
-    repo: projects.torsion.org/borgmatic-collective/borgmatic
-    tags: docs
-    dockerfile: docs/Dockerfile
+    IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs
+  commands:
+    - podman login --username "$USERNAME" --password "$PASSWORD" projects.torsion.org
+    - podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" .
+    - podman push "$IMAGE_NAME"
 
 trigger:
   repo:
     - borgmatic-collective/borgmatic
   branch:
-    - master
+    - main
   event:
     - push

NEWS

@@ -1,3 +1,54 @@
+1.7.15
+ * #326: Add configuration options and command-line flags for backing up a database from one
+   location while restoring it somewhere else.
+ * #399: Add a documentation troubleshooting note for MySQL/MariaDB authentication errors.
+ * #529: Remove upgrade-borgmatic-config command for upgrading borgmatic 1.1.0 INI-style
+   configuration.
+ * #529: Deprecate generate-borgmatic-config in favor of new "config generate" action.
+ * #529: Deprecate validate-borgmatic-config in favor of new "config validate" action.
+ * #697, #712, #716: Extract borgmatic configuration from backup via new "config bootstrap"
+   action—even when borgmatic has no configuration yet!
+ * #669: Add sample systemd user service for running borgmatic as a non-root user.
+ * #711, #713: Fix an error when "data" check time files are accessed without getting upgraded
+   first.
+
+1.7.14
+ * #484: Add a new verbosity level (-2) to disable output entirely (for console, syslog, log file,
+   or monitoring), so not even errors are shown.
+ * #688: Tweak archive check probing logic to use the newest timestamp found when multiple exist.
+ * #659: Add Borg 2 date-based matching flags to various actions for archive selection.
+ * #703: Fix an error when loading the configuration schema on Fedora Linux.
+ * #704: Fix "check" action error when repository and archive checks are configured but the archive
+   check gets skipped due to the configured frequency.
+ * #706: Fix "--archive latest" on "list" and "info" actions that only worked on the first of
+   multiple configured repositories.
+
+1.7.13
+ * #375: Restore particular PostgreSQL schemas from a database dump via "borgmatic restore --schema"
+   flag. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#restore-particular-schemas
+ * #678: Fix error from PostgreSQL when dumping a database with a "format" of "plain".
+ * #678: Fix PostgreSQL hook to support "psql_command" and "pg_restore_command" options containing
+   commands with arguments.
+ * #678: Fix calls to psql in PostgreSQL hook to ignore "~/.psqlrc", whose settings can break
+   database dumping.
+ * #680: Add support for logging each log line as a JSON object via global "--log-json" flag.
+ * #682: Fix "source_directories_must_exist" option to expand globs and tildes in source directories.
+ * #684: Rename "master" development branch to "main" to use more inclusive language. You'll need to
+   update your development checkouts accordingly.
+ * #686: Add fish shell completion script so you can tab-complete on the borgmatic command-line. See
+   the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/set-up-backups/#shell-completion
+ * #687: Fix borgmatic error when not finding the configuration schema for certain "pip install
+   --editable" development installs.
+ * #688: Fix archive checks being skipped even when particular archives haven't been checked
+   recently. This occurred when using multiple borgmatic configuration files with different
+   "archive_name_format"s, for instance.
+ * #691: Fix error in "borgmatic restore" action when the configured repository path is relative
+   instead of absolute.
+ * #694: Run "borgmatic borg" action without capturing output so interactive prompts and flags like
+   "--progress" still work.
+
 1.7.12
  * #413: Add "log_file" context to command hooks so your scripts can consume the borgmatic log file.
    See the documentation for more information:
@@ -362,7 +413,7 @@
  * #398: Clarify canonical home of borgmatic in documentation.
  * #406: Clarify that spaces in path names should not be backslashed in path names.
  * #423: Fix error handling to error loudly when Borg gets killed due to running out of memory!
- * Fix build so as not to attempt to build and push documentation for a non-master branch.
+ * Fix build so as not to attempt to build and push documentation for a non-main branch.
  * "Fix" build failure with Alpine Edge by switching from Edge to Alpine 3.13.
  * Move #borgmatic IRC channel from Freenode to Libera Chat due to Freenode takeover drama.
    IRC connection info: https://torsion.org/borgmatic/#issues

README.md

@@ -11,7 +11,7 @@ borgmatic is simple, configuration-driven backup software for servers and
 workstations. Protect your files with client-side encryption. Backup your
 databases too. Monitor it all with integrated third-party services.
 
-The canonical home of borgmatic is at <a href="https://torsion.org/borgmatic">https://torsion.org/borgmatic</a>.
+The canonical home of borgmatic is at <a href="https://torsion.org/borgmatic">https://torsion.org/borgmatic/</a>
 
 Here's an example configuration file:
 
@@ -121,8 +121,10 @@ issues.
 
 ### Social
 
-Check out the [Borg subreddit](https://www.reddit.com/r/BorgBackup/) for
-general Borg and borgmatic discussion and support.
+~~Check out the [Borg subreddit](https://www.reddit.com/r/BorgBackup/) for
+general Borg and borgmatic discussion and support.~~ borgmatic supports the
+ongoing [Reddit user
+protests](https://www.theverge.com/2023/6/10/23756476/reddit-protest-api-changes-apollo-third-party-apps).
 
 Also follow [borgmatic on Mastodon](https://fosstodon.org/@borgmatic).
 
@@ -165,5 +167,5 @@ Also, please check out the [borgmatic development
 how-to](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) for
 info on cloning source code, running tests, etc.
 
-<a href="https://build.torsion.org/borgmatic-collective/borgmatic" alt="build status">![Build Status](https://build.torsion.org/api/badges/borgmatic-collective/borgmatic/status.svg?ref=refs/heads/master)</a>
+<a href="https://build.torsion.org/borgmatic-collective/borgmatic" alt="build status">![Build Status](https://build.torsion.org/api/badges/borgmatic-collective/borgmatic/status.svg?ref=refs/heads/main)</a>
 

SECURITY.md

@@ -7,8 +7,8 @@ permalink: security-policy/index.html
 
 While we want to hear about security vulnerabilities in all versions of
 borgmatic, security fixes are only made to the most recently released version.
-It's simply not practical for our small volunteer effort to maintain multiple
-release branches and put out separate security patches for each.
+It's not practical for our small volunteer effort to maintain multiple release
+branches and put out separate security patches for each.
 
 ## Reporting a vulnerability
 

borgmatic/actions/arguments.py

@@ -0,0 +1,9 @@
+import argparse
+
+
+def update_arguments(arguments, **updates):
+    '''
+    Given an argparse.Namespace instance of command-line arguments and one or more keyword argument
+    updates to perform, return a copy of the arguments with those updates applied.
+    '''
+    return argparse.Namespace(**dict(vars(arguments), **updates))

borgmatic/actions/borg.py

@@ -8,7 +8,13 @@ logger = logging.getLogger(__name__)
 
 
 def run_borg(
-    repository, storage, local_borg_version, borg_arguments, local_path, remote_path,
+    repository,
+    storage,
+    local_borg_version,
+    borg_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
 ):
     '''
     Run the "borg" action for the given repository.
@@ -16,12 +22,15 @@ def run_borg(
     if borg_arguments.repository is None or borgmatic.config.validate.repositories_match(
         repository, borg_arguments.repository
     ):
-        logger.info(f'{repository["path"]}: Running arbitrary Borg command')
+        logger.info(
+            f'{repository.get("label", repository["path"])}: Running arbitrary Borg command'
+        )
         archive_name = borgmatic.borg.rlist.resolve_archive_name(
             repository['path'],
             borg_arguments.archive,
             storage,
             local_borg_version,
+            global_arguments,
             local_path,
             remote_path,
         )

borgmatic/actions/break_lock.py

@@ -7,7 +7,13 @@ logger = logging.getLogger(__name__)
 
 
 def run_break_lock(
-    repository, storage, local_borg_version, break_lock_arguments, local_path, remote_path,
+    repository,
+    storage,
+    local_borg_version,
+    break_lock_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
 ):
     '''
     Run the "break-lock" action for the given repository.
@@ -15,11 +21,14 @@ def run_break_lock(
     if break_lock_arguments.repository is None or borgmatic.config.validate.repositories_match(
         repository, break_lock_arguments.repository
     ):
-        logger.info(f'{repository["path"]}: Breaking repository and cache locks')
+        logger.info(
+            f'{repository.get("label", repository["path"])}: Breaking repository and cache locks'
+        )
         borgmatic.borg.break_lock.break_lock(
             repository['path'],
             storage,
             local_borg_version,
+            global_arguments,
             local_path=local_path,
             remote_path=remote_path,
         )

borgmatic/actions/check.py

@@ -37,13 +37,14 @@ def run_check(
         global_arguments.dry_run,
         **hook_context,
     )
-    logger.info(f'{repository["path"]}: Running consistency checks')
+    logger.info(f'{repository.get("label", repository["path"])}: Running consistency checks')
     borgmatic.borg.check.check_archives(
         repository['path'],
         location,
         storage,
         consistency,
         local_borg_version,
+        global_arguments,
         local_path=local_path,
         remote_path=remote_path,
         progress=check_arguments.progress,

borgmatic/actions/compact.py

@@ -39,12 +39,15 @@ def run_compact(
         **hook_context,
     )
     if borgmatic.borg.feature.available(borgmatic.borg.feature.Feature.COMPACT, local_borg_version):
-        logger.info(f'{repository["path"]}: Compacting segments{dry_run_label}')
+        logger.info(
+            f'{repository.get("label", repository["path"])}: Compacting segments{dry_run_label}'
+        )
         borgmatic.borg.compact.compact_segments(
             global_arguments.dry_run,
             repository['path'],
             storage,
             local_borg_version,
+            global_arguments,
             local_path=local_path,
             remote_path=remote_path,
             progress=compact_arguments.progress,
@@ -52,7 +55,9 @@ def run_compact(
             threshold=compact_arguments.threshold,
         )
     else:  # pragma: nocover
-        logger.info(f'{repository["path"]}: Skipping compact (only available/needed in Borg 1.2+)')
+        logger.info(
+            f'{repository.get("label", repository["path"])}: Skipping compact (only available/needed in Borg 1.2+)'
+        )
     borgmatic.hooks.command.execute_hook(
         hooks.get('after_compact'),
         hooks.get('umask'),

borgmatic/actions/config/bootstrap.py

@@ -0,0 +1,105 @@
+import json
+import logging
+import os
+
+import borgmatic.borg.extract
+import borgmatic.borg.rlist
+import borgmatic.config.validate
+import borgmatic.hooks.command
+from borgmatic.borg.state import DEFAULT_BORGMATIC_SOURCE_DIRECTORY
+
+logger = logging.getLogger(__name__)
+
+
+def get_config_paths(bootstrap_arguments, global_arguments, local_borg_version):
+    '''
+    Given:
+    The bootstrap arguments, which include the repository and archive name, borgmatic source directory,
+    destination directory, and whether to strip components.
+    The global arguments, which include the dry run flag
+    and the local borg version,
+    Return:
+    The config paths from the manifest.json file in the borgmatic source directory after extracting it from the
+    repository.
+
+    Raise ValueError if the manifest JSON is missing, can't be decoded, or doesn't contain the
+    expected configuration path data.
+    '''
+    borgmatic_source_directory = (
+        bootstrap_arguments.borgmatic_source_directory or DEFAULT_BORGMATIC_SOURCE_DIRECTORY
+    )
+    borgmatic_manifest_path = os.path.expanduser(
+        os.path.join(borgmatic_source_directory, 'bootstrap', 'manifest.json')
+    )
+    extract_process = borgmatic.borg.extract.extract_archive(
+        global_arguments.dry_run,
+        bootstrap_arguments.repository,
+        borgmatic.borg.rlist.resolve_archive_name(
+            bootstrap_arguments.repository,
+            bootstrap_arguments.archive,
+            {},
+            local_borg_version,
+            global_arguments,
+        ),
+        [borgmatic_manifest_path],
+        {},
+        {},
+        local_borg_version,
+        global_arguments,
+        extract_to_stdout=True,
+    )
+
+    manifest_json = extract_process.stdout.read()
+    if not manifest_json:
+        raise ValueError(
+            'Cannot read configuration paths from archive due to missing bootstrap manifest'
+        )
+
+    try:
+        manifest_data = json.loads(manifest_json)
+    except json.JSONDecodeError as error:
+        raise ValueError(
+            f'Cannot read configuration paths from archive due to invalid bootstrap manifest JSON: {error}'
+        )
+
+    try:
+        return manifest_data['config_paths']
+    except KeyError:
+        raise ValueError(
+            'Cannot read configuration paths from archive due to invalid bootstrap manifest'
+        )
+
+
+def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version):
+    '''
+    Run the "bootstrap" action for the given repository.
+
+    Raise ValueError if the bootstrap configuration could not be loaded.
+    Raise CalledProcessError or OSError if Borg could not be run.
+    '''
+    manifest_config_paths = get_config_paths(
+        bootstrap_arguments, global_arguments, local_borg_version
+    )
+
+    logger.info(f"Bootstrapping config paths: {', '.join(manifest_config_paths)}")
+
+    borgmatic.borg.extract.extract_archive(
+        global_arguments.dry_run,
+        bootstrap_arguments.repository,
+        borgmatic.borg.rlist.resolve_archive_name(
+            bootstrap_arguments.repository,
+            bootstrap_arguments.archive,
+            {},
+            local_borg_version,
+            global_arguments,
+        ),
+        [config_path.lstrip(os.path.sep) for config_path in manifest_config_paths],
+        {},
+        {},
+        local_borg_version,
+        global_arguments,
+        extract_to_stdout=False,
+        destination_path=bootstrap_arguments.destination,
+        strip_components=bootstrap_arguments.strip_components,
+        progress=bootstrap_arguments.progress,
+    )

borgmatic/actions/config/generate.py

@@ -0,0 +1,46 @@
+import logging
+
+import borgmatic.config.generate
+import borgmatic.config.validate
+
+logger = logging.getLogger(__name__)
+
+
+def run_generate(generate_arguments, global_arguments):
+    '''
+    Given the generate arguments and the global arguments, each as an argparse.Namespace instance,
+    run the "generate" action.
+
+    Raise FileExistsError if a file already exists at the destination path and the generate
+    arguments do not have overwrite set.
+    '''
+    dry_run_label = ' (dry run; not actually writing anything)' if global_arguments.dry_run else ''
+
+    logger.answer(
+        f'Generating a configuration file at: {generate_arguments.destination_filename}{dry_run_label}'
+    )
+
+    borgmatic.config.generate.generate_sample_configuration(
+        global_arguments.dry_run,
+        generate_arguments.source_filename,
+        generate_arguments.destination_filename,
+        borgmatic.config.validate.schema_filename(),
+        overwrite=generate_arguments.overwrite,
+    )
+
+    if generate_arguments.source_filename:
+        logger.answer(
+            f'''
+Merged in the contents of configuration file at: {generate_arguments.source_filename}
+To review the changes made, run:
+
+    diff --unified {generate_arguments.source_filename} {generate_arguments.destination_filename}'''
+        )
+
+    logger.answer(
+        '''
+This includes all available configuration options with example values, the few
+required options as indicated. Please edit the file to suit your needs.
+
+If you ever need help: https://torsion.org/borgmatic/#issues'''
+    )

borgmatic/actions/config/validate.py

@@ -0,0 +1,22 @@
+import logging
+
+import borgmatic.config.generate
+
+logger = logging.getLogger(__name__)
+
+
+def run_validate(validate_arguments, configs):
+    '''
+    Given the validate arguments as an argparse.Namespace instance and a dict of configuration
+    filename to corresponding parsed configuration, run the "validate" action.
+
+    Most of the validation is actually performed implicitly by the standard borgmatic configuration
+    loading machinery prior to here, so this function mainly exists to support additional validate
+    flags like "--show".
+    '''
+    if validate_arguments.show:
+        for config_path, config in configs.items():
+            if len(configs) > 1:
+                logger.answer('---')
+
+            logger.answer(borgmatic.config.generate.render_configuration(config))

borgmatic/actions/create.py

@@ -1,7 +1,14 @@
 import json
 import logging
+import os
+
+try:
+    import importlib_metadata
+except ModuleNotFoundError:  # pragma: nocover
+    import importlib.metadata as importlib_metadata
 
 import borgmatic.borg.create
+import borgmatic.borg.state
 import borgmatic.config.validate
 import borgmatic.hooks.command
 import borgmatic.hooks.dispatch
@@ -10,6 +17,35 @@ import borgmatic.hooks.dump
 logger = logging.getLogger(__name__)
 
 
+def create_borgmatic_manifest(location, config_paths, dry_run):
+    '''
+    Create a borgmatic manifest file to store the paths to the configuration files used to create
+    the archive.
+    '''
+    if dry_run:
+        return
+
+    borgmatic_source_directory = location.get(
+        'borgmatic_source_directory', borgmatic.borg.state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
+    )
+
+    borgmatic_manifest_path = os.path.expanduser(
+        os.path.join(borgmatic_source_directory, 'bootstrap', 'manifest.json')
+    )
+
+    if not os.path.exists(borgmatic_manifest_path):
+        os.makedirs(os.path.dirname(borgmatic_manifest_path), exist_ok=True)
+
+    with open(borgmatic_manifest_path, 'w') as config_list_file:
+        json.dump(
+            {
+                'borgmatic_version': importlib_metadata.version('borgmatic'),
+                'config_paths': config_paths,
+            },
+            config_list_file,
+        )
+
+
 def run_create(
     config_filename,
     repository,
@@ -42,7 +78,7 @@ def run_create(
         global_arguments.dry_run,
         **hook_context,
     )
-    logger.info(f'{repository["path"]}: Creating archive{dry_run_label}')
+    logger.info(f'{repository.get("label", repository["path"])}: Creating archive{dry_run_label}')
     borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
         'remove_database_dumps',
         hooks,
@@ -59,6 +95,9 @@ def run_create(
         location,
         global_arguments.dry_run,
     )
+    create_borgmatic_manifest(
+        location, global_arguments.used_config_paths, global_arguments.dry_run
+    )
     stream_processes = [process for processes in active_dumps.values() for process in processes]
 
     json_output = borgmatic.borg.create.create_archive(
@@ -67,6 +106,7 @@ def run_create(
         location,
         storage,
         local_borg_version,
+        global_arguments,
         local_path=local_path,
         remote_path=remote_path,
         progress=create_arguments.progress,

borgmatic/actions/export_tar.py

@@ -33,6 +33,7 @@ def run_export_tar(
                 export_tar_arguments.archive,
                 storage,
                 local_borg_version,
+                global_arguments,
                 local_path,
                 remote_path,
             ),
@@ -40,6 +41,7 @@ def run_export_tar(
             export_tar_arguments.destination,
             storage,
             local_borg_version,
+            global_arguments,
             local_path=local_path,
             remote_path=remote_path,
             tar_filter=export_tar_arguments.tar_filter,

borgmatic/actions/extract.py

@@ -35,7 +35,9 @@ def run_extract(
     if extract_arguments.repository is None or borgmatic.config.validate.repositories_match(
         repository, extract_arguments.repository
     ):
-        logger.info(f'{repository["path"]}: Extracting archive {extract_arguments.archive}')
+        logger.info(
+            f'{repository.get("label", repository["path"])}: Extracting archive {extract_arguments.archive}'
+        )
         borgmatic.borg.extract.extract_archive(
             global_arguments.dry_run,
             repository['path'],
@@ -44,6 +46,7 @@ def run_extract(
                 extract_arguments.archive,
                 storage,
                 local_borg_version,
+                global_arguments,
                 local_path,
                 remote_path,
             ),
@@ -51,6 +54,7 @@ def run_extract(
             location,
             storage,
             local_borg_version,
+            global_arguments,
             local_path=local_path,
             remote_path=remote_path,
             destination_path=extract_arguments.destination,

borgmatic/actions/info.py

@@ -1,6 +1,7 @@
 import json
 import logging
 
+import borgmatic.actions.arguments
 import borgmatic.borg.info
 import borgmatic.borg.rlist
 import borgmatic.config.validate
@@ -9,7 +10,13 @@ logger = logging.getLogger(__name__)
 
 
 def run_info(
-    repository, storage, local_borg_version, info_arguments, local_path, remote_path,
+    repository,
+    storage,
+    local_borg_version,
+    info_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
 ):
     '''
     Run the "info" action for the given repository and archive.
@@ -20,12 +27,15 @@ def run_info(
         repository, info_arguments.repository
     ):
         if not info_arguments.json:  # pragma: nocover
-            logger.answer(f'{repository["path"]}: Displaying archive summary information')
-        info_arguments.archive = borgmatic.borg.rlist.resolve_archive_name(
+            logger.answer(
+                f'{repository.get("label", repository["path"])}: Displaying archive summary information'
+            )
+        archive_name = borgmatic.borg.rlist.resolve_archive_name(
             repository['path'],
             info_arguments.archive,
             storage,
             local_borg_version,
+            global_arguments,
             local_path,
             remote_path,
         )
@@ -33,9 +43,10 @@ def run_info(
             repository['path'],
             storage,
             local_borg_version,
-            info_arguments=info_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
+            borgmatic.actions.arguments.update_arguments(info_arguments, archive=archive_name),
+            global_arguments,
+            local_path,
+            remote_path,
         )
         if json_output:  # pragma: nocover
             yield json.loads(json_output)

borgmatic/actions/list.py

@@ -1,6 +1,7 @@
 import json
 import logging
 
+import borgmatic.actions.arguments
 import borgmatic.borg.list
 import borgmatic.config.validate
 
@@ -8,7 +9,13 @@ logger = logging.getLogger(__name__)
 
 
 def run_list(
-    repository, storage, local_borg_version, list_arguments, local_path, remote_path,
+    repository,
+    storage,
+    local_borg_version,
+    list_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
 ):
     '''
     Run the "list" action for the given repository and archive.
@@ -20,14 +27,16 @@ def run_list(
     ):
         if not list_arguments.json:  # pragma: nocover
             if list_arguments.find_paths:
-                logger.answer(f'{repository["path"]}: Searching archives')
+                logger.answer(f'{repository.get("label", repository["path"])}: Searching archives')
             elif not list_arguments.archive:
-                logger.answer(f'{repository["path"]}: Listing archives')
-        list_arguments.archive = borgmatic.borg.rlist.resolve_archive_name(
+                logger.answer(f'{repository.get("label", repository["path"])}: Listing archives')
+
+        archive_name = borgmatic.borg.rlist.resolve_archive_name(
             repository['path'],
             list_arguments.archive,
             storage,
             local_borg_version,
+            global_arguments,
             local_path,
             remote_path,
         )
@@ -35,9 +44,10 @@ def run_list(
             repository['path'],
             storage,
             local_borg_version,
-            list_arguments=list_arguments,
-            local_path=local_path,
-            remote_path=remote_path,
+            borgmatic.actions.arguments.update_arguments(list_arguments, archive=archive_name),
+            global_arguments,
+            local_path,
+            remote_path,
         )
         if json_output:  # pragma: nocover
             yield json.loads(json_output)

borgmatic/actions/mount.py

@@ -8,7 +8,13 @@ logger = logging.getLogger(__name__)
 
 
 def run_mount(
-    repository, storage, local_borg_version, mount_arguments, local_path, remote_path,
+    repository,
+    storage,
+    local_borg_version,
+    mount_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
 ):
     '''
     Run the "mount" action for the given repository.
@@ -17,9 +23,11 @@ def run_mount(
         repository, mount_arguments.repository
     ):
         if mount_arguments.archive:
-            logger.info(f'{repository["path"]}: Mounting archive {mount_arguments.archive}')
+            logger.info(
+                f'{repository.get("label", repository["path"])}: Mounting archive {mount_arguments.archive}'
+            )
         else:  # pragma: nocover
-            logger.info(f'{repository["path"]}: Mounting repository')
+            logger.info(f'{repository.get("label", repository["path"])}: Mounting repository')
 
         borgmatic.borg.mount.mount_archive(
             repository['path'],
@@ -28,15 +36,14 @@ def run_mount(
                 mount_arguments.archive,
                 storage,
                 local_borg_version,
+                global_arguments,
                 local_path,
                 remote_path,
             ),
-            mount_arguments.mount_point,
-            mount_arguments.paths,
-            mount_arguments.foreground,
-            mount_arguments.options,
+            mount_arguments,
             storage,
             local_borg_version,
+            global_arguments,
             local_path=local_path,
             remote_path=remote_path,
         )

borgmatic/actions/prune.py

@@ -37,17 +37,17 @@ def run_prune(
         global_arguments.dry_run,
         **hook_context,
     )
-    logger.info(f'{repository["path"]}: Pruning archives{dry_run_label}')
+    logger.info(f'{repository.get("label", repository["path"])}: Pruning archives{dry_run_label}')
     borgmatic.borg.prune.prune_archives(
         global_arguments.dry_run,
         repository['path'],
         storage,
         retention,
         local_borg_version,
+        prune_arguments,
+        global_arguments,
         local_path=local_path,
         remote_path=remote_path,
-        stats=prune_arguments.stats,
-        list_archives=prune_arguments.list_archives,
     )
     borgmatic.hooks.command.execute_hook(
         hooks.get('after_prune'),

borgmatic/actions/rcreate.py

@@ -23,12 +23,13 @@ def run_rcreate(
     ):
         return
 
-    logger.info(f'{repository["path"]}: Creating repository')
+    logger.info(f'{repository.get("label", repository["path"])}: Creating repository')
     borgmatic.borg.rcreate.create_repository(
         global_arguments.dry_run,
         repository['path'],
         storage,
         local_borg_version,
+        global_arguments,
         rcreate_arguments.encryption_mode,
         rcreate_arguments.source_repository,
         rcreate_arguments.copy_crypt_key,

borgmatic/actions/restore.py

@@ -68,17 +68,21 @@ def restore_single_database(
     archive_name,
     hook_name,
     database,
+    connection_params,
 ):  # pragma: no cover
     '''
-    Given (among other things) an archive name, a database hook name, and a configured database
+    Given (among other things) an archive name, a database hook name, the hostname,
+    port, username and password as connection params, and a configured database
     configuration dict, restore that database from the archive.
     '''
-    logger.info(f'{repository}: Restoring database {database["name"]}')
+    logger.info(
+        f'{repository.get("label", repository["path"])}: Restoring database {database["name"]}'
+    )
 
     dump_pattern = borgmatic.hooks.dispatch.call_hooks(
         'make_database_dump_pattern',
         hooks,
-        repository,
+        repository['path'],
         borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
         location,
         database['name'],
@@ -87,12 +91,13 @@ def restore_single_database(
     # Kick off a single database extract to stdout.
     extract_process = borgmatic.borg.extract.extract_archive(
         dry_run=global_arguments.dry_run,
-        repository=repository,
+        repository=repository['path'],
         archive=archive_name,
         paths=borgmatic.hooks.dump.convert_glob_patterns_to_borg_patterns([dump_pattern]),
         location_config=location,
         storage_config=storage,
         local_borg_version=local_borg_version,
+        global_arguments=global_arguments,
         local_path=local_path,
         remote_path=remote_path,
         destination_path='/',
@@ -105,22 +110,30 @@ def restore_single_database(
     borgmatic.hooks.dispatch.call_hooks(
         'restore_database_dump',
         {hook_name: [database]},
-        repository,
+        repository['path'],
         borgmatic.hooks.dump.DATABASE_HOOK_NAMES,
         location,
         global_arguments.dry_run,
         extract_process,
+        connection_params,
     )
 
 
 def collect_archive_database_names(
-    repository, archive, location, storage, local_borg_version, local_path, remote_path,
+    repository,
+    archive,
+    location,
+    storage,
+    local_borg_version,
+    global_arguments,
+    local_path,
+    remote_path,
 ):
     '''
     Given a local or remote repository path, a resolved archive name, a location configuration dict,
-    a storage configuration dict, the local Borg version, and local and remote Borg paths, query the
-    archive for the names of databases it contains and return them as a dict from hook name to a
-    sequence of database names.
+    a storage configuration dict, the local Borg version, global_arguments an argparse.Namespace,
+    and local and remote Borg paths, query the archive for the names of databases it contains and
+    return them as a dict from hook name to a sequence of database names.
     '''
     borgmatic_source_directory = os.path.expanduser(
         location.get(
@@ -135,6 +148,7 @@ def collect_archive_database_names(
         archive,
         storage,
         local_borg_version,
+        global_arguments,
         list_path=parent_dump_path,
         local_path=local_path,
         remote_path=remote_path,
@@ -180,7 +194,7 @@ def find_databases_to_restore(requested_database_names, archive_database_names):
     if 'all' in restore_names[UNSPECIFIED_HOOK]:
         restore_names[UNSPECIFIED_HOOK].remove('all')
 
-        for (hook_name, database_names) in archive_database_names.items():
+        for hook_name, database_names in archive_database_names.items():
             restore_names.setdefault(hook_name, []).extend(database_names)
 
             # If a database is to be restored as part of "all", then remove it from restore names so
@@ -256,7 +270,7 @@ def run_restore(
         return
 
     logger.info(
-        f'{repository["path"]}: Restoring databases from archive {restore_arguments.archive}'
+        f'{repository.get("label", repository["path"])}: Restoring databases from archive {restore_arguments.archive}'
     )
 
     borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
@@ -273,6 +287,7 @@ def run_restore(
         restore_arguments.archive,
         storage,
         local_borg_version,
+        global_arguments,
         local_path,
         remote_path,
     )
@@ -282,12 +297,20 @@ def run_restore(
         location,
         storage,
         local_borg_version,
+        global_arguments,
         local_path,
         remote_path,
     )
     restore_names = find_databases_to_restore(restore_arguments.databases, archive_database_names)
     found_names = set()
     remaining_restore_names = {}
+    connection_params = {
+        'hostname': restore_arguments.hostname,
+        'port': restore_arguments.port,
+        'username': restore_arguments.username,
+        'password': restore_arguments.password,
+        'restore_path': restore_arguments.restore_path,
+    }
 
     for hook_name, database_names in restore_names.items():
         for database_name in database_names:
@@ -303,7 +326,7 @@ def run_restore(
 
             found_names.add(database_name)
             restore_single_database(
-                repository['path'],
+                repository,
                 location,
                 storage,
                 hooks,
@@ -313,7 +336,8 @@ def run_restore(
                 remote_path,
                 archive_name,
                 found_hook_name or hook_name,
-                found_database,
+                dict(found_database, **{'schemas': restore_arguments.schemas}),
+                connection_params,
             )
 
     # For any database that weren't found via exact matches in the hooks configuration, try to
@@ -332,7 +356,7 @@ def run_restore(
             database['name'] = database_name
 
             restore_single_database(
-                repository['path'],
+                repository,
                 location,
                 storage,
                 hooks,
@@ -342,7 +366,8 @@ def run_restore(
                 remote_path,
                 archive_name,
                 found_hook_name or hook_name,
-                database,
+                dict(database, **{'schemas': restore_arguments.schemas}),
+                connection_params,
             )
 
     borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(

borgmatic/actions/rinfo.py

@@ -8,7 +8,13 @@ logger = logging.getLogger(__name__)
 
 
 def run_rinfo(
-    repository, storage, local_borg_version, rinfo_arguments, local_path, remote_path,
+    repository,
+    storage,
+    local_borg_version,
+    rinfo_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
 ):
     '''
     Run the "rinfo" action for the given repository.
@@ -19,13 +25,16 @@ def run_rinfo(
         repository, rinfo_arguments.repository
     ):
         if not rinfo_arguments.json:  # pragma: nocover
-            logger.answer(f'{repository["path"]}: Displaying repository summary information')
+            logger.answer(
+                f'{repository.get("label", repository["path"])}: Displaying repository summary information'
+            )
 
         json_output = borgmatic.borg.rinfo.display_repository_info(
             repository['path'],
             storage,
             local_borg_version,
             rinfo_arguments=rinfo_arguments,
+            global_arguments=global_arguments,
             local_path=local_path,
             remote_path=remote_path,
         )

borgmatic/actions/rlist.py

@@ -8,7 +8,13 @@ logger = logging.getLogger(__name__)
 
 
 def run_rlist(
-    repository, storage, local_borg_version, rlist_arguments, local_path, remote_path,
+    repository,
+    storage,
+    local_borg_version,
+    rlist_arguments,
+    global_arguments,
+    local_path,
+    remote_path,
 ):
     '''
     Run the "rlist" action for the given repository.
@@ -19,13 +25,14 @@ def run_rlist(
         repository, rlist_arguments.repository
     ):
         if not rlist_arguments.json:  # pragma: nocover
-            logger.answer(f'{repository["path"]}: Listing repository')
+            logger.answer(f'{repository.get("label", repository["path"])}: Listing repository')
 
         json_output = borgmatic.borg.rlist.list_repository(
             repository['path'],
             storage,
             local_borg_version,
             rlist_arguments=rlist_arguments,
+            global_arguments=global_arguments,
             local_path=local_path,
             remote_path=remote_path,
         )

borgmatic/actions/transfer.py

@@ -17,13 +17,16 @@ def run_transfer(
     '''
     Run the "transfer" action for the given repository.
     '''
-    logger.info(f'{repository["path"]}: Transferring archives to repository')
+    logger.info(
+        f'{repository.get("label", repository["path"])}: Transferring archives to repository'
+    )
     borgmatic.borg.transfer.transfer_archives(
         global_arguments.dry_run,
         repository['path'],
         storage,
         local_borg_version,
         transfer_arguments,
+        global_arguments,
         local_path=local_path,
         remote_path=remote_path,
     )

borgmatic/borg/borg.py

@@ -1,8 +1,9 @@
 import logging
 
+import borgmatic.commands.arguments
 import borgmatic.logger
 from borgmatic.borg import environment, flags
-from borgmatic.execute import execute_command
+from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
 logger = logging.getLogger(__name__)
 
@@ -36,6 +37,11 @@ def run_arbitrary_borg(
         command_options_start_index = 2 if options[0] in BORG_SUBCOMMANDS_WITH_SUBCOMMANDS else 1
         borg_command = tuple(options[:command_options_start_index])
         command_options = tuple(options[command_options_start_index:])
+
+        if borg_command and borg_command[0] in borgmatic.commands.arguments.ACTION_ALIASES.keys():
+            logger.warning(
+                f"Borg's {borg_command[0]} subcommand is supported natively by borgmatic. Try this instead: borgmatic {borg_command[0]}"
+            )
     except IndexError:
         borg_command = ()
         command_options = ()
@@ -62,7 +68,7 @@ def run_arbitrary_borg(
 
     return execute_command(
         full_command,
-        output_log_level=logging.ANSWER,
+        output_file=DO_NOT_CAPTURE,
         borg_local_path=local_path,
         extra_environment=environment.make_environment(storage_config),
     )

borgmatic/borg/break_lock.py

@@ -7,12 +7,17 @@ logger = logging.getLogger(__name__)
 
 
 def break_lock(
-    repository_path, storage_config, local_borg_version, local_path='borg', remote_path=None,
+    repository_path,
+    storage_config,
+    local_borg_version,
+    global_arguments,
+    local_path='borg',
+    remote_path=None,
 ):
     '''
     Given a local or remote repository path, a storage configuration dict, the local Borg version,
-    and optional local and remote Borg paths, break any repository and cache locks leftover from Borg
-    aborting.
+    an argparse.Namespace of global arguments, and optional local and remote Borg paths, break any
+    repository and cache locks leftover from Borg aborting.
     '''
     umask = storage_config.get('umask', None)
     lock_wait = storage_config.get('lock_wait', None)
@@ -21,6 +26,7 @@ def break_lock(
         (local_path, 'break-lock')
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
+        + (('--log-json',) if global_arguments.log_json else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())

borgmatic/borg/check.py

@@ -1,5 +1,7 @@
 import argparse
 import datetime
+import hashlib
+import itertools
 import json
 import logging
 import os
@@ -88,12 +90,18 @@ def parse_frequency(frequency):
 
 
 def filter_checks_on_frequency(
-    location_config, consistency_config, borg_repository_id, checks, force
+    location_config,
+    consistency_config,
+    borg_repository_id,
+    checks,
+    force,
+    archives_check_id=None,
 ):
     '''
     Given a location config, a consistency config with a "checks" sequence of dicts, a Borg
-    repository ID, a sequence of checks, and whether to force checks to run, filter down those
-    checks based on the configured "frequency" for each check as compared to its check time file.
+    repository ID, a sequence of checks, whether to force checks to run, and an ID for the archives
+    check potentially being run (if any), filter down those checks based on the configured
+    "frequency" for each check as compared to its check time file.
 
     In other words, a check whose check time file's timestamp is too new (based on the configured
     frequency) will get cut from the returned sequence of checks. Example:
@@ -127,8 +135,8 @@ def filter_checks_on_frequency(
         if not frequency_delta:
             continue
 
-        check_time = read_check_time(
-            make_check_time_path(location_config, borg_repository_id, check)
+        check_time = probe_for_check_time(
+            location_config, borg_repository_id, check, archives_check_id
         )
         if not check_time:
             continue
@@ -145,36 +153,19 @@ def filter_checks_on_frequency(
     return tuple(filtered_checks)
 
 
-def make_check_flags(local_borg_version, storage_config, checks, check_last=None, prefix=None):
+def make_archive_filter_flags(
+    local_borg_version, storage_config, checks, check_last=None, prefix=None
+):
     '''
     Given the local Borg version, a storage configuration dict, a parsed sequence of checks, the
     check last value, and a consistency check prefix, transform the checks into tuple of
-    command-line flags.
+    command-line flags for filtering archives in a check command.
 
-    For example, given parsed checks of:
-
-        ('repository',)
-
-    This will be returned as:
-
-        ('--repository-only',)
-
-    However, if both "repository" and "archives" are in checks, then omit them from the returned
-    flags because Borg does both checks by default. If "data" is in checks, that implies "archives".
-
-    Additionally, if a check_last value is given and "archives" is in checks, then include a
-    "--last" flag. And if a prefix value is given and "archives" is in checks, then include a
-    "--match-archives" flag.
+    If a check_last value is given and "archives" is in checks, then include a "--last" flag. And if
+    a prefix value is given and "archives" is in checks, then include a "--match-archives" flag.
     '''
-    if 'data' in checks:
-        data_flags = ('--verify-data',)
-        checks += ('archives',)
-    else:
-        data_flags = ()
-
-    if 'archives' in checks:
-        last_flags = ('--last', str(check_last)) if check_last else ()
-        match_archives_flags = (
+    if 'archives' in checks or 'data' in checks:
+        return (('--last', str(check_last)) if check_last else ()) + (
             (
                 ('--match-archives', f'sh:{prefix}*')
                 if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version)
@@ -189,19 +180,53 @@ def make_check_flags(local_borg_version, storage_config, checks, check_last=None
                 )
             )
         )
-    else:
-        last_flags = ()
-        match_archives_flags = ()
-        if check_last:
-            logger.warning(
-                'Ignoring check_last option, as "archives" or "data" are not in consistency checks'
-            )
-        if prefix:
-            logger.warning(
-                'Ignoring consistency prefix option, as "archives" or "data" are not in consistency checks'
-            )
 
-    common_flags = last_flags + match_archives_flags + data_flags
+    if check_last:
+        logger.warning(
+            'Ignoring check_last option, as "archives" or "data" are not in consistency checks'
+        )
+    if prefix:
+        logger.warning(
+            'Ignoring consistency prefix option, as "archives" or "data" are not in consistency checks'
+        )
+
+    return ()
+
+
+def make_archives_check_id(archive_filter_flags):
+    '''
+    Given a sequence of flags to filter archives, return a unique hash corresponding to those
+    particular flags. If there are no flags, return None.
+    '''
+    if not archive_filter_flags:
+        return None
+
+    return hashlib.sha256(' '.join(archive_filter_flags).encode()).hexdigest()
+
+
+def make_check_flags(checks, archive_filter_flags):
+    '''
+    Given a parsed sequence of checks and a sequence of flags to filter archives, transform the
+    checks into tuple of command-line check flags.
+
+    For example, given parsed checks of:
+
+        ('repository',)
+
+    This will be returned as:
+
+        ('--repository-only',)
+
+    However, if both "repository" and "archives" are in checks, then omit them from the returned
+    flags because Borg does both checks by default. If "data" is in checks, that implies "archives".
+    '''
+    if 'data' in checks:
+        data_flags = ('--verify-data',)
+        checks += ('archives',)
+    else:
+        data_flags = ()
+
+    common_flags = (archive_filter_flags if 'archives' in checks else ()) + data_flags
 
     if {'repository', 'archives'}.issubset(set(checks)):
         return common_flags
@@ -212,18 +237,27 @@ def make_check_flags(local_borg_version, storage_config, checks, check_last=None
     )
 
 
-def make_check_time_path(location_config, borg_repository_id, check_type):
+def make_check_time_path(location_config, borg_repository_id, check_type, archives_check_id=None):
     '''
-    Given a location configuration dict, a Borg repository ID, and the name of a check type
-    ("repository", "archives", etc.), return a path for recording that check's time (the time of
-    that check last occurring).
+    Given a location configuration dict, a Borg repository ID, the name of a check type
+    ("repository", "archives", etc.), and a unique hash of the archives filter flags, return a
+    path for recording that check's time (the time of that check last occurring).
     '''
+    borgmatic_source_directory = os.path.expanduser(
+        location_config.get('borgmatic_source_directory', state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY)
+    )
+
+    if check_type in ('archives', 'data'):
+        return os.path.join(
+            borgmatic_source_directory,
+            'checks',
+            borg_repository_id,
+            check_type,
+            archives_check_id if archives_check_id else 'all',
+        )
+
     return os.path.join(
-        os.path.expanduser(
-            location_config.get(
-                'borgmatic_source_directory', state.DEFAULT_BORGMATIC_SOURCE_DIRECTORY
-            )
-        ),
+        borgmatic_source_directory,
         'checks',
         borg_repository_id,
         check_type,
@@ -253,12 +287,81 @@ def read_check_time(path):
         return None
 
 
+def probe_for_check_time(location_config, borg_repository_id, check, archives_check_id):
+    '''
+    Given a location configuration dict, a Borg repository ID, the name of a check type
+    ("repository", "archives", etc.), and a unique hash of the archives filter flags, return a
+    the corresponding check time or None if such a check time does not exist.
+
+    When the check type is "archives" or "data", this function probes two different paths to find
+    the check time, e.g.:
+
+      ~/.borgmatic/checks/1234567890/archives/9876543210
+      ~/.borgmatic/checks/1234567890/archives/all
+
+    ... and returns the maximum modification time of the files found (if any). The first path
+    represents a more specific archives check time (a check on a subset of archives), and the second
+    is a fallback to the last "all" archives check.
+
+    For other check types, this function reads from a single check time path, e.g.:
+
+      ~/.borgmatic/checks/1234567890/repository
+    '''
+    check_times = (
+        read_check_time(group[0])
+        for group in itertools.groupby(
+            (
+                make_check_time_path(location_config, borg_repository_id, check, archives_check_id),
+                make_check_time_path(location_config, borg_repository_id, check),
+            )
+        )
+    )
+
+    try:
+        return max(check_time for check_time in check_times if check_time)
+    except ValueError:
+        return None
+
+
+def upgrade_check_times(location_config, borg_repository_id):
+    '''
+    Given a location configuration dict and a Borg repository ID, upgrade any corresponding check
+    times on disk from old-style paths to new-style paths.
+
+    Currently, the only upgrade performed is renaming an archive or data check path that looks like:
+
+      ~/.borgmatic/checks/1234567890/archives
+
+    to:
+
+      ~/.borgmatic/checks/1234567890/archives/all
+    '''
+    for check_type in ('archives', 'data'):
+        new_path = make_check_time_path(location_config, borg_repository_id, check_type, 'all')
+        old_path = os.path.dirname(new_path)
+        temporary_path = f'{old_path}.temp'
+
+        if not os.path.isfile(old_path) and not os.path.isfile(temporary_path):
+            continue
+
+        logger.debug(f'Upgrading archives check time from {old_path} to {new_path}')
+
+        try:
+            os.rename(old_path, temporary_path)
+        except FileNotFoundError:
+            pass
+
+        os.mkdir(old_path)
+        os.rename(temporary_path, new_path)
+
+
 def check_archives(
     repository_path,
     location_config,
     storage_config,
     consistency_config,
     local_borg_version,
+    global_arguments,
     local_path='borg',
     remote_path=None,
     progress=None,
@@ -283,6 +386,7 @@ def check_archives(
                 storage_config,
                 local_borg_version,
                 argparse.Namespace(json=True),
+                global_arguments,
                 local_path,
                 remote_path,
             )
@@ -290,16 +394,26 @@ def check_archives(
     except (json.JSONDecodeError, KeyError):
         raise ValueError(f'Cannot determine Borg repository ID for {repository_path}')
 
+    upgrade_check_times(location_config, borg_repository_id)
+
+    check_last = consistency_config.get('check_last', None)
+    prefix = consistency_config.get('prefix')
+    configured_checks = parse_checks(consistency_config, only_checks)
+    lock_wait = None
+    extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '')
+    archive_filter_flags = make_archive_filter_flags(
+        local_borg_version, storage_config, configured_checks, check_last, prefix
+    )
+    archives_check_id = make_archives_check_id(archive_filter_flags)
+
     checks = filter_checks_on_frequency(
         location_config,
         consistency_config,
         borg_repository_id,
-        parse_checks(consistency_config, only_checks),
+        configured_checks,
         force,
+        archives_check_id,
     )
-    check_last = consistency_config.get('check_last', None)
-    lock_wait = None
-    extra_borg_options = storage_config.get('extra_borg_options', {}).get('check', '')
 
     if set(checks).intersection({'repository', 'archives', 'data'}):
         lock_wait = storage_config.get('lock_wait')
@@ -310,13 +424,12 @@ def check_archives(
         if logger.isEnabledFor(logging.DEBUG):
             verbosity_flags = ('--debug', '--show-rc')
 
-        prefix = consistency_config.get('prefix')
-
         full_command = (
             (local_path, 'check')
             + (('--repair',) if repair else ())
-            + make_check_flags(local_borg_version, storage_config, checks, check_last, prefix)
+            + make_check_flags(checks, archive_filter_flags)
             + (('--remote-path', remote_path) if remote_path else ())
+            + (('--log-json',) if global_arguments.log_json else ())
             + (('--lock-wait', str(lock_wait)) if lock_wait else ())
             + verbosity_flags
             + (('--progress',) if progress else ())
@@ -336,10 +449,18 @@ def check_archives(
             execute_command(full_command, extra_environment=borg_environment)
 
         for check in checks:
-            write_check_time(make_check_time_path(location_config, borg_repository_id, check))
+            write_check_time(
+                make_check_time_path(location_config, borg_repository_id, check, archives_check_id)
+            )
 
     if 'extract' in checks:
         extract.extract_last_archive_dry_run(
-            storage_config, local_borg_version, repository_path, lock_wait, local_path, remote_path
+            storage_config,
+            local_borg_version,
+            global_arguments,
+            repository_path,
+            lock_wait,
+            local_path,
+            remote_path,
         )
         write_check_time(make_check_time_path(location_config, borg_repository_id, 'extract'))

borgmatic/borg/compact.py

@@ -11,6 +11,7 @@ def compact_segments(
     repository_path,
     storage_config,
     local_borg_version,
+    global_arguments,
     local_path='borg',
     remote_path=None,
     progress=False,
@@ -29,6 +30,7 @@ def compact_segments(
         (local_path, 'compact')
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
+        + (('--log-json',) if global_arguments.log_json else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--progress',) if progress else ())
         + (('--cleanup-commits',) if cleanup_commits else ())

borgmatic/borg/create.py

@@ -314,7 +314,7 @@ def check_all_source_directories_exist(source_directories):
     missing_directories = [
         source_directory
         for source_directory in source_directories
-        if not os.path.exists(source_directory)
+        if not all([os.path.exists(directory) for directory in expand_directory(source_directory)])
     ]
     if missing_directories:
         raise ValueError(f"Source directories do not exist: {', '.join(missing_directories)}")
@@ -326,6 +326,7 @@ def create_archive(
     location_config,
     storage_config,
     local_borg_version,
+    global_arguments,
     local_path='borg',
     remote_path=None,
     progress=False,
@@ -350,7 +351,9 @@ def create_archive(
     sources = deduplicate_directories(
         map_directories_to_devices(
             expand_directories(
-                tuple(location_config.get('source_directories', ())) + borgmatic_source_directories
+                tuple(location_config.get('source_directories', ()))
+                + borgmatic_source_directories
+                + tuple(global_arguments.used_config_paths)
             )
         ),
         additional_directory_devices=map_directories_to_devices(
@@ -438,6 +441,7 @@ def create_archive(
         + (('--files-cache', files_cache) if files_cache else ())
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
+        + (('--log-json',) if global_arguments.log_json else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (
             ('--list', '--filter', list_filter_flags)
@@ -509,7 +513,9 @@ def create_archive(
         )
     elif output_log_level is None:
         return execute_command_and_capture_output(
-            create_command, working_directory=working_directory, extra_environment=borg_environment,
+            create_command,
+            working_directory=working_directory,
+            extra_environment=borg_environment,
         )
     else:
         execute_command(

borgmatic/borg/export_tar.py

@@ -15,6 +15,7 @@ def export_tar_archive(
     destination_path,
     storage_config,
     local_borg_version,
+    global_arguments,
     local_path='borg',
     remote_path=None,
     tar_filter=None,
@@ -38,6 +39,7 @@ def export_tar_archive(
         (local_path, 'export-tar')
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
+        + (('--log-json',) if global_arguments.log_json else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--list',) if list_files else ())
@@ -45,7 +47,11 @@ def export_tar_archive(
         + (('--dry-run',) if dry_run else ())
         + (('--tar-filter', tar_filter) if tar_filter else ())
         + (('--strip-components', str(strip_components)) if strip_components else ())
-        + flags.make_repository_archive_flags(repository_path, archive, local_borg_version,)
+        + flags.make_repository_archive_flags(
+            repository_path,
+            archive,
+            local_borg_version,
+        )
         + (destination_path,)
         + (tuple(paths) if paths else ())
     )

borgmatic/borg/extract.py

@@ -2,6 +2,7 @@ import logging
 import os
 import subprocess
 
+import borgmatic.config.validate
 from borgmatic.borg import environment, feature, flags, rlist
 from borgmatic.execute import DO_NOT_CAPTURE, execute_command
 
@@ -11,6 +12,7 @@ logger = logging.getLogger(__name__)
 def extract_last_archive_dry_run(
     storage_config,
     local_borg_version,
+    global_arguments,
     repository_path,
     lock_wait=None,
     local_path='borg',
@@ -20,8 +22,6 @@ def extract_last_archive_dry_run(
     Perform an extraction dry-run of the most recent archive. If there are no archives, skip the
     dry-run.
     '''
-    remote_path_flags = ('--remote-path', remote_path) if remote_path else ()
-    lock_wait_flags = ('--lock-wait', str(lock_wait)) if lock_wait else ()
     verbosity_flags = ()
     if logger.isEnabledFor(logging.DEBUG):
         verbosity_flags = ('--debug', '--show-rc')
@@ -30,7 +30,13 @@ def extract_last_archive_dry_run(
 
     try:
         last_archive_name = rlist.resolve_archive_name(
-            repository_path, 'latest', storage_config, local_borg_version, local_path, remote_path
+            repository_path,
+            'latest',
+            storage_config,
+            local_borg_version,
+            global_arguments,
+            local_path,
+            remote_path,
         )
     except ValueError:
         logger.warning('No archives found. Skipping extract consistency check.')
@@ -40,8 +46,9 @@ def extract_last_archive_dry_run(
     borg_environment = environment.make_environment(storage_config)
     full_extract_command = (
         (local_path, 'extract', '--dry-run')
-        + remote_path_flags
-        + lock_wait_flags
+        + (('--remote-path', remote_path) if remote_path else ())
+        + (('--log-json',) if global_arguments.log_json else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + verbosity_flags
         + list_flag
         + flags.make_repository_archive_flags(
@@ -62,6 +69,7 @@ def extract_archive(
     location_config,
     storage_config,
     local_borg_version,
+    global_arguments,
     local_path='borg',
     remote_path=None,
     destination_path=None,
@@ -71,9 +79,9 @@ def extract_archive(
 ):
     '''
     Given a dry-run flag, a local or remote repository path, an archive name, zero or more paths to
-    restore from the archive, the local Borg version string, location/storage configuration dicts,
-    optional local and remote Borg paths, and an optional destination path to extract to, extract
-    the archive into the current directory.
+    restore from the archive, the local Borg version string, an argparse.Namespace of global
+    arguments, location/storage configuration dicts, optional local and remote Borg paths, and an
+    optional destination path to extract to, extract the archive into the current directory.
 
     If extract to stdout is True, then start the extraction streaming to stdout, and return that
     extract process as an instance of subprocess.Popen.
@@ -101,6 +109,7 @@ def extract_archive(
         + (('--remote-path', remote_path) if remote_path else ())
         + numeric_ids_flags
         + (('--umask', str(umask)) if umask else ())
+        + (('--log-json',) if global_arguments.log_json else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
@@ -108,7 +117,13 @@ def extract_archive(
         + (('--strip-components', str(strip_components)) if strip_components else ())
         + (('--progress',) if progress else ())
         + (('--stdout',) if extract_to_stdout else ())
-        + flags.make_repository_archive_flags(repository, archive, local_borg_version,)
+        + flags.make_repository_archive_flags(
+            # Make the repository path absolute so the working directory changes below don't
+            # prevent Borg from finding the repo.
+            borgmatic.config.validate.normalize_repository_path(repository),
+            archive,
+            local_borg_version,
+        )
         + (tuple(paths) if paths else ())
     )
 

borgmatic/borg/feature.py

@@ -1,6 +1,6 @@
 from enum import Enum
 
-from pkg_resources import parse_version
+from packaging.version import parse
 
 
 class Feature(Enum):
@@ -18,17 +18,17 @@ class Feature(Enum):
 
 
 FEATURE_TO_MINIMUM_BORG_VERSION = {
-    Feature.COMPACT: parse_version('1.2.0a2'),  # borg compact
-    Feature.ATIME: parse_version('1.2.0a7'),  # borg create --atime
-    Feature.NOFLAGS: parse_version('1.2.0a8'),  # borg create --noflags
-    Feature.NUMERIC_IDS: parse_version('1.2.0b3'),  # borg create/extract/mount --numeric-ids
-    Feature.UPLOAD_RATELIMIT: parse_version('1.2.0b3'),  # borg create --upload-ratelimit
-    Feature.SEPARATE_REPOSITORY_ARCHIVE: parse_version('2.0.0a2'),  # --repo with separate archive
-    Feature.RCREATE: parse_version('2.0.0a2'),  # borg rcreate
-    Feature.RLIST: parse_version('2.0.0a2'),  # borg rlist
-    Feature.RINFO: parse_version('2.0.0a2'),  # borg rinfo
-    Feature.MATCH_ARCHIVES: parse_version('2.0.0b3'),  # borg --match-archives
-    Feature.EXCLUDED_FILES_MINUS: parse_version('2.0.0b5'),  # --list --filter uses "-" for excludes
+    Feature.COMPACT: parse('1.2.0a2'),  # borg compact
+    Feature.ATIME: parse('1.2.0a7'),  # borg create --atime
+    Feature.NOFLAGS: parse('1.2.0a8'),  # borg create --noflags
+    Feature.NUMERIC_IDS: parse('1.2.0b3'),  # borg create/extract/mount --numeric-ids
+    Feature.UPLOAD_RATELIMIT: parse('1.2.0b3'),  # borg create --upload-ratelimit
+    Feature.SEPARATE_REPOSITORY_ARCHIVE: parse('2.0.0a2'),  # --repo with separate archive
+    Feature.RCREATE: parse('2.0.0a2'),  # borg rcreate
+    Feature.RLIST: parse('2.0.0a2'),  # borg rlist
+    Feature.RINFO: parse('2.0.0a2'),  # borg rinfo
+    Feature.MATCH_ARCHIVES: parse('2.0.0b3'),  # borg --match-archives
+    Feature.EXCLUDED_FILES_MINUS: parse('2.0.0b5'),  # --list --filter uses "-" for excludes
 }
 
 
@@ -37,4 +37,4 @@ def available(feature, borg_version):
     Given a Borg Feature constant and a Borg version string, return whether that feature is
     available in that version of Borg.
     '''
-    return FEATURE_TO_MINIMUM_BORG_VERSION[feature] <= parse_version(borg_version)
+    return FEATURE_TO_MINIMUM_BORG_VERSION[feature] <= parse(borg_version)

borgmatic/borg/info.py

@@ -12,13 +12,14 @@ def display_archives_info(
     storage_config,
     local_borg_version,
     info_arguments,
+    global_arguments,
     local_path='borg',
     remote_path=None,
 ):
     '''
-    Given a local or remote repository path, a storage config dict, the local Borg version, and the
-    arguments to the info action, display summary information for Borg archives in the repository or
-    return JSON summary information.
+    Given a local or remote repository path, a storage config dict, the local Borg version, global
+    arguments as an argparse.Namespace, and the arguments to the info action, display summary
+    information for Borg archives in the repository or return JSON summary information.
     '''
     borgmatic.logger.add_custom_log_levels()
     lock_wait = storage_config.get('lock_wait', None)
@@ -36,6 +37,7 @@ def display_archives_info(
             else ()
         )
         + flags.make_flags('remote-path', remote_path)
+        + flags.make_flags('log-json', global_arguments.log_json)
         + flags.make_flags('lock-wait', lock_wait)
         + (
             (
@@ -62,7 +64,8 @@ def display_archives_info(
 
     if info_arguments.json:
         return execute_command_and_capture_output(
-            full_command, extra_environment=environment.make_environment(storage_config),
+            full_command,
+            extra_environment=environment.make_environment(storage_config),
         )
     else:
         execute_command(

borgmatic/borg/list.py

@@ -25,6 +25,7 @@ def make_list_command(
     storage_config,
     local_borg_version,
     list_arguments,
+    global_arguments,
     local_path='borg',
     remote_path=None,
 ):
@@ -48,6 +49,7 @@ def make_list_command(
             else ()
         )
         + flags.make_flags('remote-path', remote_path)
+        + flags.make_flags('log-json', global_arguments.log_json)
         + flags.make_flags('lock-wait', lock_wait)
         + flags.make_flags_from_arguments(list_arguments, excludes=MAKE_FLAGS_EXCLUDES)
         + (
@@ -90,14 +92,16 @@ def capture_archive_listing(
     archive,
     storage_config,
     local_borg_version,
+    global_arguments,
     list_path=None,
     local_path='borg',
     remote_path=None,
 ):
     '''
     Given a local or remote repository path, an archive name, a storage config dict, the local Borg
-    version, the archive path in which to list files, and local and remote Borg paths, capture the
-    output of listing that archive and return it as a list of file paths.
+    version, global arguments as an argparse.Namespace, the archive path in which to list files, and
+    local and remote Borg paths, capture the output of listing that archive and return it as a list
+    of file paths.
     '''
     borg_environment = environment.make_environment(storage_config)
 
@@ -115,6 +119,7 @@ def capture_archive_listing(
                     json=None,
                     format='{path}{NL}',  # noqa: FS003
                 ),
+                global_arguments,
                 local_path,
                 remote_path,
             ),
@@ -130,15 +135,17 @@ def list_archive(
     storage_config,
     local_borg_version,
     list_arguments,
+    global_arguments,
     local_path='borg',
     remote_path=None,
 ):
     '''
-    Given a local or remote repository path, a storage config dict, the local Borg version, the
-    arguments to the list action, and local and remote Borg paths, display the output of listing
-    the files of a Borg archive (or return JSON output). If list_arguments.find_paths are given,
-    list the files by searching across multiple archives. If neither find_paths nor archive name
-    are given, instead list the archives in the given repository.
+    Given a local or remote repository path, a storage config dict, the local Borg version, global
+    arguments as an argparse.Namespace, the arguments to the list action as an argparse.Namespace,
+    and local and remote Borg paths, display the output of listing the files of a Borg archive (or
+    return JSON output). If list_arguments.find_paths are given, list the files by searching across
+    multiple archives. If neither find_paths nor archive name are given, instead list the archives
+    in the given repository.
     '''
     borgmatic.logger.add_custom_log_levels()
 
@@ -164,6 +171,7 @@ def list_archive(
             storage_config,
             local_borg_version,
             rlist_arguments,
+            global_arguments,
             local_path,
             remote_path,
         )
@@ -205,6 +213,7 @@ def list_archive(
                     storage_config,
                     local_borg_version,
                     rlist_arguments,
+                    global_arguments,
                     local_path,
                     remote_path,
                 ),
@@ -233,6 +242,7 @@ def list_archive(
             storage_config,
             local_borg_version,
             archive_arguments,
+            global_arguments,
             local_path,
             remote_path,
         ) + make_find_paths(list_arguments.find_paths)

borgmatic/borg/mount.py

@@ -9,20 +9,18 @@ logger = logging.getLogger(__name__)
 def mount_archive(
     repository_path,
     archive,
-    mount_point,
-    paths,
-    foreground,
-    options,
+    mount_arguments,
     storage_config,
     local_borg_version,
+    global_arguments,
     local_path='borg',
     remote_path=None,
 ):
     '''
     Given a local or remote repository path, an optional archive name, a filesystem mount point,
     zero or more paths to mount from the archive, extra Borg mount options, a storage configuration
-    dict, the local Borg version, and optional local and remote Borg paths, mount the archive onto
-    the mount point.
+    dict, the local Borg version, global arguments as an argparse.Namespace instance, and optional
+    local and remote Borg paths, mount the archive onto the mount point.
     '''
     umask = storage_config.get('umask', None)
     lock_wait = storage_config.get('lock_wait', None)
@@ -31,11 +29,15 @@ def mount_archive(
         (local_path, 'mount')
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
+        + (('--log-json',) if global_arguments.log_json else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
-        + (('--foreground',) if foreground else ())
-        + (('-o', options) if options else ())
+        + flags.make_flags_from_arguments(
+            mount_arguments,
+            excludes=('repository', 'archive', 'mount_point', 'paths', 'options'),
+        )
+        + (('-o', mount_arguments.options) if mount_arguments.options else ())
         + (
             (
                 flags.make_repository_flags(repository_path, local_borg_version)
@@ -52,14 +54,14 @@ def mount_archive(
                 else flags.make_repository_flags(repository_path, local_borg_version)
             )
         )
-        + (mount_point,)
-        + (tuple(paths) if paths else ())
+        + (mount_arguments.mount_point,)
+        + (tuple(mount_arguments.paths) if mount_arguments.paths else ())
     )
 
     borg_environment = environment.make_environment(storage_config)
 
     # Don't capture the output when foreground mode is used so that ctrl-C can work properly.
-    if foreground:
+    if mount_arguments.foreground:
         execute_command(
             full_command,
             output_file=DO_NOT_CAPTURE,

borgmatic/borg/prune.py

@@ -53,10 +53,10 @@ def prune_archives(
     storage_config,
     retention_config,
     local_borg_version,
+    prune_arguments,
+    global_arguments,
     local_path='borg',
     remote_path=None,
-    stats=False,
-    list_archives=False,
 ):
     '''
     Given dry-run flag, a local or remote repository path, a storage config dict, and a
@@ -73,17 +73,22 @@ def prune_archives(
         + make_prune_flags(storage_config, retention_config, local_borg_version)
         + (('--remote-path', remote_path) if remote_path else ())
         + (('--umask', str(umask)) if umask else ())
+        + (('--log-json',) if global_arguments.log_json else ())
         + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + (('--stats',) if stats and not dry_run else ())
+        + (('--stats',) if prune_arguments.stats and not dry_run else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
-        + (('--list',) if list_archives else ())
+        + flags.make_flags_from_arguments(
+            prune_arguments,
+            excludes=('repository', 'stats', 'list_archives'),
+        )
+        + (('--list',) if prune_arguments.list_archives else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--dry-run',) if dry_run else ())
         + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
-    if stats or list_archives:
+    if prune_arguments.stats or prune_arguments.list_archives:
         output_log_level = logging.ANSWER
     else:
         output_log_level = logging.INFO

borgmatic/borg/rcreate.py

@@ -16,6 +16,7 @@ def create_repository(
     repository_path,
     storage_config,
     local_borg_version,
+    global_arguments,
     encryption_mode,
     source_repository=None,
     copy_crypt_key=False,
@@ -37,6 +38,7 @@ def create_repository(
             storage_config,
             local_borg_version,
             argparse.Namespace(json=True),
+            global_arguments,
             local_path,
             remote_path,
         )
@@ -46,6 +48,7 @@ def create_repository(
         if error.returncode != RINFO_REPOSITORY_NOT_FOUND_EXIT_CODE:
             raise
 
+    lock_wait = storage_config.get('lock_wait')
     extra_borg_options = storage_config.get('extra_borg_options', {}).get('rcreate', '')
 
     rcreate_command = (
@@ -63,6 +66,8 @@ def create_repository(
         + (('--make-parent-dirs',) if make_parent_dirs else ())
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ())
+        + (('--log-json',) if global_arguments.log_json else ())
+        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
         + (('--remote-path', remote_path) if remote_path else ())
         + (tuple(extra_borg_options.split(' ')) if extra_borg_options else ())
         + flags.make_repository_flags(repository_path, local_borg_version)

borgmatic/borg/rinfo.py

@@ -12,13 +12,14 @@ def display_repository_info(
     storage_config,
     local_borg_version,
     rinfo_arguments,
+    global_arguments,
     local_path='borg',
     remote_path=None,
 ):
     '''
-    Given a local or remote repository path, a storage config dict, the local Borg version, and the
-    arguments to the rinfo action, display summary information for the Borg repository or return
-    JSON summary information.
+    Given a local or remote repository path, a storage config dict, the local Borg version, the
+    arguments to the rinfo action, and global arguments as an argparse.Namespace, display summary
+    information for the Borg repository or return JSON summary information.
     '''
     borgmatic.logger.add_custom_log_levels()
     lock_wait = storage_config.get('lock_wait', None)
@@ -41,6 +42,7 @@ def display_repository_info(
             else ()
         )
         + flags.make_flags('remote-path', remote_path)
+        + flags.make_flags('log-json', global_arguments.log_json)
         + flags.make_flags('lock-wait', lock_wait)
         + (('--json',) if rinfo_arguments.json else ())
         + flags.make_repository_flags(repository_path, local_borg_version)
@@ -50,7 +52,8 @@ def display_repository_info(
 
     if rinfo_arguments.json:
         return execute_command_and_capture_output(
-            full_command, extra_environment=extra_environment,
+            full_command,
+            extra_environment=extra_environment,
         )
     else:
         execute_command(

borgmatic/borg/rlist.py

@@ -12,35 +12,37 @@ def resolve_archive_name(
     archive,
     storage_config,
     local_borg_version,
+    global_arguments,
     local_path='borg',
     remote_path=None,
 ):
     '''
-    Given a local or remote repository path, an archive name, a storage config dict, a local Borg
-    path, and a remote Borg path, simply return the archive name. But if the archive name is
-    "latest", then instead introspect the repository for the latest archive and return its name.
+    Given a local or remote repository path, an archive name, a storage config dict, the local Borg
+    version, global arguments as an argparse.Namespace, a local Borg path, and a remote Borg path,
+    return the archive name. But if the archive name is "latest", then instead introspect the
+    repository for the latest archive and return its name.
 
     Raise ValueError if "latest" is given but there are no archives in the repository.
     '''
     if archive != 'latest':
         return archive
 
-    lock_wait = storage_config.get('lock_wait', None)
-
     full_command = (
         (
             local_path,
             'rlist' if feature.available(feature.Feature.RLIST, local_borg_version) else 'list',
         )
         + flags.make_flags('remote-path', remote_path)
-        + flags.make_flags('lock-wait', lock_wait)
+        + flags.make_flags('log-json', global_arguments.log_json)
+        + flags.make_flags('lock-wait', storage_config.get('lock_wait'))
         + flags.make_flags('last', 1)
         + ('--short',)
         + flags.make_repository_flags(repository_path, local_borg_version)
     )
 
     output = execute_command_and_capture_output(
-        full_command, extra_environment=environment.make_environment(storage_config),
+        full_command,
+        extra_environment=environment.make_environment(storage_config),
     )
     try:
         latest_archive = output.strip().splitlines()[-1]
@@ -60,16 +62,15 @@ def make_rlist_command(
     storage_config,
     local_borg_version,
     rlist_arguments,
+    global_arguments,
     local_path='borg',
     remote_path=None,
 ):
     '''
     Given a local or remote repository path, a storage config dict, the local Borg version, the
-    arguments to the rlist action, and local and remote Borg paths, return a command as a tuple to
-    list archives with a repository.
+    arguments to the rlist action, global arguments as an argparse.Namespace instance, and local and
+    remote Borg paths, return a command as a tuple to list archives with a repository.
     '''
-    lock_wait = storage_config.get('lock_wait', None)
-
     return (
         (
             local_path,
@@ -86,7 +87,8 @@ def make_rlist_command(
             else ()
         )
         + flags.make_flags('remote-path', remote_path)
-        + flags.make_flags('lock-wait', lock_wait)
+        + flags.make_flags('log-json', global_arguments.log_json)
+        + flags.make_flags('lock-wait', storage_config.get('lock_wait'))
         + (
             (
                 flags.make_flags('match-archives', f'sh:{rlist_arguments.prefix}*')
@@ -112,13 +114,15 @@ def list_repository(
     storage_config,
     local_borg_version,
     rlist_arguments,
+    global_arguments,
     local_path='borg',
     remote_path=None,
 ):
     '''
     Given a local or remote repository path, a storage config dict, the local Borg version, the
-    arguments to the list action, and local and remote Borg paths, display the output of listing
-    Borg archives in the given repository (or return JSON output).
+    arguments to the list action, global arguments as an argparse.Namespace instance, and local and
+    remote Borg paths, display the output of listing Borg archives in the given repository (or
+    return JSON output).
     '''
     borgmatic.logger.add_custom_log_levels()
     borg_environment = environment.make_environment(storage_config)
@@ -128,6 +132,7 @@ def list_repository(
         storage_config,
         local_borg_version,
         rlist_arguments,
+        global_arguments,
         local_path,
         remote_path,
     )

borgmatic/borg/transfer.py

@@ -13,12 +13,14 @@ def transfer_archives(
     storage_config,
     local_borg_version,
     transfer_arguments,
+    global_arguments,
     local_path='borg',
     remote_path=None,
 ):
     '''
     Given a dry-run flag, a local or remote repository path, a storage config dict, the local Borg
-    version, and the arguments to the transfer action, transfer archives to the given repository.
+    version, the arguments to the transfer action, and global arguments as an argparse.Namespace
+    instance, transfer archives to the given repository.
     '''
     borgmatic.logger.add_custom_log_levels()
 
@@ -27,6 +29,7 @@ def transfer_archives(
         + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + flags.make_flags('remote-path', remote_path)
+        + flags.make_flags('log-json', global_arguments.log_json)
         + flags.make_flags('lock-wait', storage_config.get('lock_wait', None))
         + (
             flags.make_flags_from_arguments(

borgmatic/borg/version.py

@@ -19,7 +19,8 @@ def local_borg_version(storage_config, local_path='borg'):
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
     )
     output = execute_command_and_capture_output(
-        full_command, extra_environment=environment.make_environment(storage_config),
+        full_command,
+        extra_environment=environment.make_environment(storage_config),
     )
 
     try:

borgmatic/commands/arguments.py

@@ -1,14 +1,17 @@
 import collections
+import itertools
+import sys
 from argparse import Action, ArgumentParser
 
 from borgmatic.config import collect
 
-SUBPARSER_ALIASES = {
+ACTION_ALIASES = {
     'rcreate': ['init', '-I'],
     'prune': ['-p'],
     'compact': [],
     'create': ['-C'],
     'check': ['-k'],
+    'config': [],
     'extract': ['-x'],
     'export-tar': [],
     'mount': ['-m'],
@@ -24,80 +27,209 @@ SUBPARSER_ALIASES = {
 }
 
 
-def parse_subparser_arguments(unparsed_arguments, subparsers):
+def get_subaction_parsers(action_parser):
     '''
-    Given a sequence of arguments and a dict from subparser name to argparse.ArgumentParser
-    instance, give each requested action's subparser a shot at parsing all arguments. This allows
-    common arguments like "--repository" to be shared across multiple subparsers.
+    Given an argparse.ArgumentParser instance, lookup the subactions in it and return a dict from
+    subaction name to subaction parser.
+    '''
+    if not action_parser._subparsers:
+        return {}
 
-    Return the result as a tuple of (a dict mapping from subparser name to a parsed namespace of
-    arguments, a list of remaining arguments not claimed by any subparser).
-    '''
-    arguments = collections.OrderedDict()
-    remaining_arguments = list(unparsed_arguments)
-    alias_to_subparser_name = {
-        alias: subparser_name
-        for subparser_name, aliases in SUBPARSER_ALIASES.items()
-        for alias in aliases
+    return {
+        subaction_name: subaction_parser
+        for group_action in action_parser._subparsers._group_actions
+        for subaction_name, subaction_parser in group_action.choices.items()
     }
 
-    # If the "borg" action is used, skip all other subparsers. This avoids confusion like
-    # "borg list" triggering borgmatic's own list action.
-    if 'borg' in unparsed_arguments:
-        subparsers = {'borg': subparsers['borg']}
 
-    for argument in remaining_arguments:
-        canonical_name = alias_to_subparser_name.get(argument, argument)
-        subparser = subparsers.get(canonical_name)
+def get_subactions_for_actions(action_parsers):
+    '''
+    Given a dict from action name to an argparse.ArgumentParser instance, make a map from action
+    name to the names of contained sub-actions.
+    '''
+    return {
+        action: tuple(
+            subaction_name
+            for group_action in action_parser._subparsers._group_actions
+            for subaction_name in group_action.choices.keys()
+        )
+        for action, action_parser in action_parsers.items()
+        if action_parser._subparsers
+    }
 
-        if not subparser:
-            continue
 
-        # If a parsed value happens to be the same as the name of a subparser, remove it from the
-        # remaining arguments. This prevents, for instance, "check --only extract" from triggering
-        # the "extract" subparser.
-        parsed, unused_remaining = subparser.parse_known_args(unparsed_arguments)
+def omit_values_colliding_with_action_names(unparsed_arguments, parsed_arguments):
+    '''
+    Given a sequence of string arguments and a dict from action name to parsed argparse.Namespace
+    arguments, return the string arguments with any values omitted that happen to be the same as
+    the name of a borgmatic action.
+
+    This prevents, for instance, "check --only extract" from triggering the "extract" action.
+    '''
+    remaining_arguments = list(unparsed_arguments)
+
+    for action_name, parsed in parsed_arguments.items():
         for value in vars(parsed).values():
             if isinstance(value, str):
-                if value in subparsers:
+                if value in ACTION_ALIASES.keys():
                     remaining_arguments.remove(value)
             elif isinstance(value, list):
                 for item in value:
-                    if item in subparsers:
+                    if item in ACTION_ALIASES.keys():
                         remaining_arguments.remove(item)
 
-        arguments[canonical_name] = parsed
+    return tuple(remaining_arguments)
 
-    # If no actions are explicitly requested, assume defaults.
-    if not arguments and '--help' not in unparsed_arguments and '-h' not in unparsed_arguments:
-        for subparser_name in ('create', 'prune', 'compact', 'check'):
-            subparser = subparsers[subparser_name]
-            parsed, unused_remaining = subparser.parse_known_args(unparsed_arguments)
-            arguments[subparser_name] = parsed
 
-    remaining_arguments = list(unparsed_arguments)
+def parse_and_record_action_arguments(
+    unparsed_arguments, parsed_arguments, action_parser, action_name, canonical_name=None
+):
+    '''
+    Given unparsed arguments as a sequence of strings, parsed arguments as a dict from action name
+    to parsed argparse.Namespace, a parser to parse with, an action name, and an optional canonical
+    action name (in case this the action name is an alias), parse the arguments and return a list of
+    any remaining string arguments that were not parsed. Also record the parsed argparse.Namespace
+    by setting it into the given parsed arguments. Return None if no parsing occurs because the
+    given action doesn't apply to the given unparsed arguments.
+    '''
+    filtered_arguments = omit_values_colliding_with_action_names(
+        unparsed_arguments, parsed_arguments
+    )
 
-    # Now ask each subparser, one by one, to greedily consume arguments.
-    for subparser_name, subparser in subparsers.items():
-        if subparser_name not in arguments.keys():
+    if action_name not in filtered_arguments:
+        return tuple(unparsed_arguments)
+
+    parsed, remaining = action_parser.parse_known_args(filtered_arguments)
+    parsed_arguments[canonical_name or action_name] = parsed
+
+    # Special case: If this is a "borg" action, greedily consume all arguments after (+1) the "borg"
+    # argument.
+    if action_name == 'borg':
+        borg_options_index = remaining.index('borg') + 1
+        parsed_arguments['borg'].options = remaining[borg_options_index:]
+        remaining = remaining[:borg_options_index]
+
+    return tuple(argument for argument in remaining if argument != action_name)
+
+
+def get_unparsable_arguments(remaining_action_arguments):
+    '''
+    Given a sequence of argument tuples (one per action parser that parsed arguments), determine the
+    remaining arguments that no action parsers have consumed.
+    '''
+    if not remaining_action_arguments:
+        return ()
+
+    return tuple(
+        argument
+        for argument in dict.fromkeys(
+            itertools.chain.from_iterable(remaining_action_arguments)
+        ).keys()
+        if all(argument in action_arguments for action_arguments in remaining_action_arguments)
+    )
+
+
+def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parser):
+    '''
+    Given a sequence of arguments, a dict from action name to argparse.ArgumentParser instance,
+    and the global parser as a argparse.ArgumentParser instance, give each requested action's
+    parser a shot at parsing all arguments. This allows common arguments like "--repository" to be
+    shared across multiple action parsers.
+
+    Return the result as a tuple of: (a dict mapping from action name to an argparse.Namespace of
+    parsed arguments, a tuple of argument tuples where each is the remaining arguments not claimed
+    by any action parser).
+    '''
+    arguments = collections.OrderedDict()
+    help_requested = bool('--help' in unparsed_arguments or '-h' in unparsed_arguments)
+    remaining_action_arguments = []
+    alias_to_action_name = {
+        alias: action_name for action_name, aliases in ACTION_ALIASES.items() for alias in aliases
+    }
+
+    # If the "borg" action is used, skip all other action parsers. This avoids confusion like
+    # "borg list" triggering borgmatic's own list action.
+    if 'borg' in unparsed_arguments:
+        action_parsers = {'borg': action_parsers['borg']}
+
+    # Ask each action parser, one by one, to parse arguments.
+    for argument in unparsed_arguments:
+        action_name = argument
+        canonical_name = alias_to_action_name.get(action_name, action_name)
+        action_parser = action_parsers.get(action_name)
+
+        if not action_parser:
             continue
 
-        subparser = subparsers[subparser_name]
-        unused_parsed, remaining_arguments = subparser.parse_known_args(remaining_arguments)
+        subaction_parsers = get_subaction_parsers(action_parser)
 
-    # Special case: If "borg" is present in the arguments, consume all arguments after (+1) the
-    # "borg" action.
-    if 'borg' in arguments:
-        borg_options_index = remaining_arguments.index('borg') + 1
-        arguments['borg'].options = remaining_arguments[borg_options_index:]
-        remaining_arguments = remaining_arguments[:borg_options_index]
+        # But first parse with subaction parsers, if any.
+        if subaction_parsers:
+            subactions_parsed = False
 
-    # Remove the subparser names themselves.
-    for subparser_name, subparser in subparsers.items():
-        if subparser_name in remaining_arguments:
-            remaining_arguments.remove(subparser_name)
+            for subaction_name, subaction_parser in subaction_parsers.items():
+                remaining_action_arguments.append(
+                    tuple(
+                        argument
+                        for argument in parse_and_record_action_arguments(
+                            unparsed_arguments,
+                            arguments,
+                            subaction_parser,
+                            subaction_name,
+                        )
+                        if argument != action_name
+                    )
+                )
 
-    return (arguments, remaining_arguments)
+                if subaction_name in arguments:
+                    subactions_parsed = True
+
+            if not subactions_parsed:
+                if help_requested:
+                    action_parser.print_help()
+                    sys.exit(0)
+                else:
+                    raise ValueError(
+                        f"Missing sub-action after {action_name} action. Expected one of: {', '.join(get_subactions_for_actions(action_parsers)[action_name])}"
+                    )
+        # Otherwise, parse with the main action parser.
+        else:
+            remaining_action_arguments.append(
+                parse_and_record_action_arguments(
+                    unparsed_arguments, arguments, action_parser, action_name, canonical_name
+                )
+            )
+
+    # If no actions were explicitly requested, assume defaults.
+    if not arguments and not help_requested:
+        for default_action_name in ('create', 'prune', 'compact', 'check'):
+            default_action_parser = action_parsers[default_action_name]
+            remaining_action_arguments.append(
+                parse_and_record_action_arguments(
+                    tuple(unparsed_arguments) + (default_action_name,),
+                    arguments,
+                    default_action_parser,
+                    default_action_name,
+                )
+            )
+
+    arguments['global'], remaining = global_parser.parse_known_args(unparsed_arguments)
+    remaining_action_arguments.append(remaining)
+
+    # Prevent action names that follow "--config" paths from being considered as additional paths.
+    for argument_name in arguments.keys():
+        if argument_name == 'global':
+            continue
+
+        for action_name in [argument_name] + ACTION_ALIASES.get(argument_name, []):
+            if action_name in arguments['global'].config_paths:
+                arguments['global'].config_paths.remove(action_name)
+                break
+
+    return (
+        arguments,
+        tuple(remaining_action_arguments) if arguments else unparsed_arguments,
+    )
 
 
 class Extend_action(Action):
@@ -109,14 +241,17 @@ class Extend_action(Action):
         items = getattr(namespace, self.dest, None)
 
         if items:
-            items.extend(values)
+            items.extend(values)  # pragma: no cover
         else:
             setattr(namespace, self.dest, list(values))
 
 
 def make_parsers():
     '''
-    Build a top-level parser and its subparsers and return them as a tuple.
+    Build a global arguments parser, individual action parsers, and a combined parser containing
+    both. Return them as a tuple. The global parser is useful for parsing just global arguments
+    while ignoring actions, and the combined parser is handy for displaying help that includes
+    everything: global flags, a list of actions, etc.
     '''
     config_paths = collect.get_default_config_paths(expand_home=True)
     unexpanded_config_paths = collect.get_default_config_paths(expand_home=False)
@@ -152,39 +287,46 @@ def make_parsers():
         '-v',
         '--verbosity',
         type=int,
-        choices=range(-1, 3),
+        choices=range(-2, 3),
         default=0,
-        help='Display verbose progress to the console (from only errors to very verbose: -1, 0, 1, or 2)',
+        help='Display verbose progress to the console (disabled, errors only, default, some, or lots: -2, -1, 0, 1, or 2)',
     )
     global_group.add_argument(
         '--syslog-verbosity',
         type=int,
-        choices=range(-1, 3),
+        choices=range(-2, 3),
         default=0,
-        help='Log verbose progress to syslog (from only errors to very verbose: -1, 0, 1, or 2). Ignored when console is interactive or --log-file is given',
+        help='Log verbose progress to syslog (disabled, errors only, default, some, or lots: -2, -1, 0, 1, or 2). Ignored when console is interactive or --log-file is given',
     )
     global_group.add_argument(
         '--log-file-verbosity',
         type=int,
-        choices=range(-1, 3),
+        choices=range(-2, 3),
         default=0,
-        help='Log verbose progress to log file (from only errors to very verbose: -1, 0, 1, or 2). Only used when --log-file is given',
+        help='Log verbose progress to log file (disabled, errors only, default, some, or lots: -2, -1, 0, 1, or 2). Only used when --log-file is given',
     )
     global_group.add_argument(
         '--monitoring-verbosity',
         type=int,
-        choices=range(-1, 3),
+        choices=range(-2, 3),
         default=0,
-        help='Log verbose progress to monitoring integrations that support logging (from only errors to very verbose: -1, 0, 1, or 2)',
+        help='Log verbose progress to monitoring integrations that support logging (from disabled, errors only, default, some, or lots: -2, -1, 0, 1, or 2)',
     )
     global_group.add_argument(
-        '--log-file', type=str, help='Write log messages to this file instead of syslog',
+        '--log-file',
+        type=str,
+        help='Write log messages to this file instead of syslog',
     )
     global_group.add_argument(
         '--log-file-format',
         type=str,
         help='Log format string used for log messages written to the log file',
     )
+    global_group.add_argument(
+        '--log-json',
+        action='store_true',
+        help='Write log messages and console output as one JSON object per log line instead of formatted text',
+    )
     global_group.add_argument(
         '--override',
         metavar='SECTION.OPTION=VALUE',
@@ -205,6 +347,12 @@ def make_parsers():
         action='store_true',
         help='Show bash completion script and exit',
     )
+    global_group.add_argument(
+        '--fish-completion',
+        default=False,
+        action='store_true',
+        help='Show fish completion script and exit',
+    )
     global_group.add_argument(
         '--version',
         dest='version',
@@ -213,7 +361,7 @@ def make_parsers():
         help='Display installed version number of borgmatic and exit',
     )
 
-    top_level_parser = ArgumentParser(
+    global_plus_action_parser = ArgumentParser(
         description='''
             Simple, configuration-driven backup software for servers and workstations. If none of
             the action options are given, then borgmatic defaults to: create, prune, compact, and
@@ -222,14 +370,14 @@ def make_parsers():
         parents=[global_parser],
     )
 
-    subparsers = top_level_parser.add_subparsers(
+    action_parsers = global_plus_action_parser.add_subparsers(
         title='actions',
         metavar='',
         help='Specify zero or more actions. Defaults to create, prune, compact, and check. Use --help with action for details:',
     )
-    rcreate_parser = subparsers.add_parser(
+    rcreate_parser = action_parsers.add_parser(
         'rcreate',
-        aliases=SUBPARSER_ALIASES['rcreate'],
+        aliases=ACTION_ALIASES['rcreate'],
         help='Create a new, empty Borg repository',
         description='Create a new, empty Borg repository',
         add_help=False,
@@ -246,7 +394,7 @@ def make_parsers():
         '--source-repository',
         '--other-repo',
         metavar='KEY_REPOSITORY',
-        help='Path to an existing Borg repository whose key material should be reused (Borg 2.x+ only)',
+        help='Path to an existing Borg repository whose key material should be reused [Borg 2.x+ only]',
     )
     rcreate_group.add_argument(
         '--repository',
@@ -255,13 +403,16 @@ def make_parsers():
     rcreate_group.add_argument(
         '--copy-crypt-key',
         action='store_true',
-        help='Copy the crypt key used for authenticated encryption from the source repository, defaults to a new random key (Borg 2.x+ only)',
+        help='Copy the crypt key used for authenticated encryption from the source repository, defaults to a new random key [Borg 2.x+ only]',
     )
     rcreate_group.add_argument(
-        '--append-only', action='store_true', help='Create an append-only repository',
+        '--append-only',
+        action='store_true',
+        help='Create an append-only repository',
     )
     rcreate_group.add_argument(
-        '--storage-quota', help='Create a repository with a fixed storage quota',
+        '--storage-quota',
+        help='Create a repository with a fixed storage quota',
     )
     rcreate_group.add_argument(
         '--make-parent-dirs',
@@ -272,11 +423,11 @@ def make_parsers():
         '-h', '--help', action='help', help='Show this help message and exit'
     )
 
-    transfer_parser = subparsers.add_parser(
+    transfer_parser = action_parsers.add_parser(
         'transfer',
-        aliases=SUBPARSER_ALIASES['transfer'],
-        help='Transfer archives from one repository to another, optionally upgrading the transferred data (Borg 2.0+ only)',
-        description='Transfer archives from one repository to another, optionally upgrading the transferred data (Borg 2.0+ only)',
+        aliases=ACTION_ALIASES['transfer'],
+        help='Transfer archives from one repository to another, optionally upgrading the transferred data [Borg 2.0+ only]',
+        description='Transfer archives from one repository to another, optionally upgrading the transferred data [Borg 2.0+ only]',
         add_help=False,
     )
     transfer_group = transfer_parser.add_argument_group('transfer arguments')
@@ -321,13 +472,33 @@ def make_parsers():
     transfer_group.add_argument(
         '--last', metavar='N', help='Only transfer last N archives after other filters are applied'
     )
+    transfer_group.add_argument(
+        '--oldest',
+        metavar='TIMESPAN',
+        help='Transfer archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]',
+    )
+    transfer_group.add_argument(
+        '--newest',
+        metavar='TIMESPAN',
+        help='Transfer archives within a time range that ends at timestamp of the newest archive and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]',
+    )
+    transfer_group.add_argument(
+        '--older',
+        metavar='TIMESPAN',
+        help='Transfer archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]',
+    )
+    transfer_group.add_argument(
+        '--newer',
+        metavar='TIMESPAN',
+        help='Transfer archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]',
+    )
     transfer_group.add_argument(
         '-h', '--help', action='help', help='Show this help message and exit'
     )
 
-    prune_parser = subparsers.add_parser(
+    prune_parser = action_parsers.add_parser(
         'prune',
-        aliases=SUBPARSER_ALIASES['prune'],
+        aliases=ACTION_ALIASES['prune'],
         help='Prune archives according to the retention policy (with Borg 1.2+, run compact afterwards to actually free space)',
         description='Prune archives according to the retention policy (with Borg 1.2+, run compact afterwards to actually free space)',
         add_help=False,
@@ -347,13 +518,33 @@ def make_parsers():
     prune_group.add_argument(
         '--list', dest='list_archives', action='store_true', help='List archives kept/pruned'
     )
+    prune_group.add_argument(
+        '--oldest',
+        metavar='TIMESPAN',
+        help='Prune archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]',
+    )
+    prune_group.add_argument(
+        '--newest',
+        metavar='TIMESPAN',
+        help='Prune archives within a time range that ends at timestamp of the newest archive and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]',
+    )
+    prune_group.add_argument(
+        '--older',
+        metavar='TIMESPAN',
+        help='Prune archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]',
+    )
+    prune_group.add_argument(
+        '--newer',
+        metavar='TIMESPAN',
+        help='Prune archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]',
+    )
     prune_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    compact_parser = subparsers.add_parser(
+    compact_parser = action_parsers.add_parser(
         'compact',
-        aliases=SUBPARSER_ALIASES['compact'],
-        help='Compact segments to free space (Borg 1.2+, borgmatic 1.5.23+ only)',
-        description='Compact segments to free space (Borg 1.2+, borgmatic 1.5.23+ only)',
+        aliases=ACTION_ALIASES['compact'],
+        help='Compact segments to free space [Borg 1.2+, borgmatic 1.5.23+ only]',
+        description='Compact segments to free space [Borg 1.2+, borgmatic 1.5.23+ only]',
         add_help=False,
     )
     compact_group = compact_parser.add_argument_group('compact arguments')
@@ -373,7 +564,7 @@ def make_parsers():
         dest='cleanup_commits',
         default=False,
         action='store_true',
-        help='Cleanup commit-only 17-byte segment files left behind by Borg 1.1 (flag in Borg 1.2 only)',
+        help='Cleanup commit-only 17-byte segment files left behind by Borg 1.1 [flag in Borg 1.2 only]',
     )
     compact_group.add_argument(
         '--threshold',
@@ -385,9 +576,9 @@ def make_parsers():
         '-h', '--help', action='help', help='Show this help message and exit'
     )
 
-    create_parser = subparsers.add_parser(
+    create_parser = action_parsers.add_parser(
         'create',
-        aliases=SUBPARSER_ALIASES['create'],
+        aliases=ACTION_ALIASES['create'],
         help='Create an archive (actually perform a backup)',
         description='Create an archive (actually perform a backup)',
         add_help=False,
@@ -419,9 +610,9 @@ def make_parsers():
     )
     create_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    check_parser = subparsers.add_parser(
+    check_parser = action_parsers.add_parser(
         'check',
-        aliases=SUBPARSER_ALIASES['check'],
+        aliases=ACTION_ALIASES['check'],
         help='Check archives for consistency',
         description='Check archives for consistency',
         add_help=False,
@@ -461,9 +652,9 @@ def make_parsers():
     )
     check_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    extract_parser = subparsers.add_parser(
+    extract_parser = action_parsers.add_parser(
         'extract',
-        aliases=SUBPARSER_ALIASES['extract'],
+        aliases=ACTION_ALIASES['extract'],
         help='Extract files from a named archive to the current directory',
         description='Extract a named archive to the current directory',
         add_help=False,
@@ -507,9 +698,118 @@ def make_parsers():
         '-h', '--help', action='help', help='Show this help message and exit'
     )
 
-    export_tar_parser = subparsers.add_parser(
+    config_parser = action_parsers.add_parser(
+        'config',
+        aliases=ACTION_ALIASES['config'],
+        help='Perform configuration file related operations',
+        description='Perform configuration file related operations',
+        add_help=False,
+    )
+
+    config_group = config_parser.add_argument_group('config arguments')
+    config_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
+
+    config_parsers = config_parser.add_subparsers(
+        title='config sub-actions',
+    )
+
+    config_bootstrap_parser = config_parsers.add_parser(
+        'bootstrap',
+        help='Extract the borgmatic configuration files from a named archive',
+        description='Extract the borgmatic configuration files from a named archive',
+        add_help=False,
+    )
+    config_bootstrap_group = config_bootstrap_parser.add_argument_group(
+        'config bootstrap arguments'
+    )
+    config_bootstrap_group.add_argument(
+        '--repository',
+        help='Path of repository to extract config files from',
+        required=True,
+    )
+    config_bootstrap_group.add_argument(
+        '--borgmatic-source-directory',
+        help='Path that stores the config files used to create an archive and additional source files used for temporary internal state like borgmatic database dumps. Defaults to ~/.borgmatic',
+    )
+    config_bootstrap_group.add_argument(
+        '--archive',
+        help='Name of archive to extract config files from, defaults to "latest"',
+        default='latest',
+    )
+    config_bootstrap_group.add_argument(
+        '--destination',
+        metavar='PATH',
+        dest='destination',
+        help='Directory to extract config files into, defaults to /',
+        default='/',
+    )
+    config_bootstrap_group.add_argument(
+        '--strip-components',
+        type=lambda number: number if number == 'all' else int(number),
+        metavar='NUMBER',
+        help='Number of leading path components to remove from each extracted path or "all" to strip all leading path components. Skip paths with fewer elements',
+    )
+    config_bootstrap_group.add_argument(
+        '--progress',
+        dest='progress',
+        default=False,
+        action='store_true',
+        help='Display progress for each file as it is extracted',
+    )
+    config_bootstrap_group.add_argument(
+        '-h', '--help', action='help', help='Show this help message and exit'
+    )
+
+    config_generate_parser = config_parsers.add_parser(
+        'generate',
+        help='Generate a sample borgmatic configuration file',
+        description='Generate a sample borgmatic configuration file',
+        add_help=False,
+    )
+    config_generate_group = config_generate_parser.add_argument_group('config generate arguments')
+    config_generate_group.add_argument(
+        '-s',
+        '--source',
+        dest='source_filename',
+        help='Optional configuration file to merge into the generated configuration, useful for upgrading your configuration',
+    )
+    config_generate_group.add_argument(
+        '-d',
+        '--destination',
+        dest='destination_filename',
+        default=config_paths[0],
+        help=f'Destination configuration file, default: {unexpanded_config_paths[0]}',
+    )
+    config_generate_group.add_argument(
+        '--overwrite',
+        default=False,
+        action='store_true',
+        help='Whether to overwrite any existing destination file, defaults to false',
+    )
+    config_generate_group.add_argument(
+        '-h', '--help', action='help', help='Show this help message and exit'
+    )
+
+    config_validate_parser = config_parsers.add_parser(
+        'validate',
+        help='Validate borgmatic configuration files specified with --config (see borgmatic --help)',
+        description='Validate borgmatic configuration files specified with --config (see borgmatic --help)',
+        add_help=False,
+    )
+    config_validate_group = config_validate_parser.add_argument_group('config validate arguments')
+    config_validate_group.add_argument(
+        '-s',
+        '--show',
+        action='store_true',
+        help='Show the validated configuration after all include merging has occurred',
+    )
+    config_validate_group.add_argument(
+        '-h', '--help', action='help', help='Show this help message and exit'
+    )
+
+    export_tar_parser = action_parsers.add_parser(
         'export-tar',
-        aliases=SUBPARSER_ALIASES['export-tar'],
+        aliases=ACTION_ALIASES['export-tar'],
         help='Export an archive to a tar-formatted file or stream',
         description='Export an archive to a tar-formatted file or stream',
         add_help=False,
@@ -553,9 +853,9 @@ def make_parsers():
         '-h', '--help', action='help', help='Show this help message and exit'
     )
 
-    mount_parser = subparsers.add_parser(
+    mount_parser = action_parsers.add_parser(
         'mount',
-        aliases=SUBPARSER_ALIASES['mount'],
+        aliases=ACTION_ALIASES['mount'],
         help='Mount files from a named archive as a FUSE filesystem',
         description='Mount a named archive as a FUSE filesystem',
         add_help=False,
@@ -587,12 +887,40 @@ def make_parsers():
         action='store_true',
         help='Stay in foreground until ctrl-C is pressed',
     )
+    mount_group.add_argument(
+        '--first',
+        metavar='N',
+        help='Mount first N archives after other filters are applied',
+    )
+    mount_group.add_argument(
+        '--last', metavar='N', help='Mount last N archives after other filters are applied'
+    )
+    mount_group.add_argument(
+        '--oldest',
+        metavar='TIMESPAN',
+        help='Mount archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]',
+    )
+    mount_group.add_argument(
+        '--newest',
+        metavar='TIMESPAN',
+        help='Mount archives within a time range that ends at timestamp of the newest archive and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]',
+    )
+    mount_group.add_argument(
+        '--older',
+        metavar='TIMESPAN',
+        help='Mount archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]',
+    )
+    mount_group.add_argument(
+        '--newer',
+        metavar='TIMESPAN',
+        help='Mount archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]',
+    )
     mount_group.add_argument('--options', dest='options', help='Extra Borg mount options')
     mount_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    umount_parser = subparsers.add_parser(
+    umount_parser = action_parsers.add_parser(
         'umount',
-        aliases=SUBPARSER_ALIASES['umount'],
+        aliases=ACTION_ALIASES['umount'],
         help='Unmount a FUSE filesystem that was mounted with "borgmatic mount"',
         description='Unmount a mounted FUSE filesystem',
         add_help=False,
@@ -607,9 +935,9 @@ def make_parsers():
     )
     umount_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    restore_parser = subparsers.add_parser(
+    restore_parser = action_parsers.add_parser(
         'restore',
-        aliases=SUBPARSER_ALIASES['restore'],
+        aliases=ACTION_ALIASES['restore'],
         help='Restore database dumps from a named archive',
         description='Restore database dumps from a named archive. (To extract files instead, use "borgmatic extract".)',
         add_help=False,
@@ -629,20 +957,48 @@ def make_parsers():
         dest='databases',
         help="Names of databases to restore from archive, defaults to all databases. Note that any databases to restore must be defined in borgmatic's configuration",
     )
+    restore_group.add_argument(
+        '--schema',
+        metavar='NAME',
+        nargs='+',
+        dest='schemas',
+        help='Names of schemas to restore from the database, defaults to all schemas. Schemas are only supported for PostgreSQL and MongoDB databases',
+    )
+    restore_group.add_argument(
+        '--hostname',
+        help='Database hostname to restore to. Defaults to the "restore_hostname" option in borgmatic\'s configuration',
+    )
+    restore_group.add_argument(
+        '--port',
+        help='Port to restore to. Defaults to the "restore_port" option in borgmatic\'s configuration',
+    )
+    restore_group.add_argument(
+        '--username',
+        help='Username with which to connect to the database. Defaults to the "restore_username" option in borgmatic\'s configuration',
+    )
+    restore_group.add_argument(
+        '--password',
+        help='Password with which to connect to the restore database. Defaults to the "restore_password" option in borgmatic\'s configuration',
+    )
+    restore_group.add_argument(
+        '--restore-path',
+        help='Path to restore SQLite database dumps to. Defaults to the "restore_path" option in borgmatic\'s configuration',
+    )
     restore_group.add_argument(
         '-h', '--help', action='help', help='Show this help message and exit'
     )
 
-    rlist_parser = subparsers.add_parser(
+    rlist_parser = action_parsers.add_parser(
         'rlist',
-        aliases=SUBPARSER_ALIASES['rlist'],
+        aliases=ACTION_ALIASES['rlist'],
         help='List repository',
         description='List the archives in a repository',
         add_help=False,
     )
     rlist_group = rlist_parser.add_argument_group('rlist arguments')
     rlist_group.add_argument(
-        '--repository', help='Path of repository to list, defaults to the configured repositories',
+        '--repository',
+        help='Path of repository to list, defaults to the configured repositories',
     )
     rlist_group.add_argument(
         '--short', default=False, action='store_true', help='Output only archive names'
@@ -670,11 +1026,31 @@ def make_parsers():
     rlist_group.add_argument(
         '--last', metavar='N', help='List last N archives after other filters are applied'
     )
+    rlist_group.add_argument(
+        '--oldest',
+        metavar='TIMESPAN',
+        help='List archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]',
+    )
+    rlist_group.add_argument(
+        '--newest',
+        metavar='TIMESPAN',
+        help='List archives within a time range that ends at timestamp of the newest archive and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]',
+    )
+    rlist_group.add_argument(
+        '--older',
+        metavar='TIMESPAN',
+        help='List archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]',
+    )
+    rlist_group.add_argument(
+        '--newer',
+        metavar='TIMESPAN',
+        help='List archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]',
+    )
     rlist_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    list_parser = subparsers.add_parser(
+    list_parser = action_parsers.add_parser(
         'list',
-        aliases=SUBPARSER_ALIASES['list'],
+        aliases=ACTION_ALIASES['list'],
         help='List archive',
         description='List the files in an archive or search for a file across archives',
         add_help=False,
@@ -745,9 +1121,9 @@ def make_parsers():
     )
     list_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    rinfo_parser = subparsers.add_parser(
+    rinfo_parser = action_parsers.add_parser(
         'rinfo',
-        aliases=SUBPARSER_ALIASES['rinfo'],
+        aliases=ACTION_ALIASES['rinfo'],
         help='Show repository summary information such as disk space used',
         description='Show repository summary information such as disk space used',
         add_help=False,
@@ -762,9 +1138,9 @@ def make_parsers():
     )
     rinfo_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    info_parser = subparsers.add_parser(
+    info_parser = action_parsers.add_parser(
         'info',
-        aliases=SUBPARSER_ALIASES['info'],
+        aliases=ACTION_ALIASES['info'],
         help='Show archive summary information such as disk space used',
         description='Show archive summary information such as disk space used',
         add_help=False,
@@ -801,11 +1177,31 @@ def make_parsers():
     info_group.add_argument(
         '--last', metavar='N', help='Show info for last N archives after other filters are applied'
     )
+    info_group.add_argument(
+        '--oldest',
+        metavar='TIMESPAN',
+        help='Show info for archives within a specified time range starting from the timestamp of the oldest archive (e.g. 7d or 12m) [Borg 2.x+ only]',
+    )
+    info_group.add_argument(
+        '--newest',
+        metavar='TIMESPAN',
+        help='Show info for archives within a time range that ends at timestamp of the newest archive and starts a specified time range ago (e.g. 7d or 12m) [Borg 2.x+ only]',
+    )
+    info_group.add_argument(
+        '--older',
+        metavar='TIMESPAN',
+        help='Show info for archives that are older than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]',
+    )
+    info_group.add_argument(
+        '--newer',
+        metavar='TIMESPAN',
+        help='Show info for archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]',
+    )
     info_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    break_lock_parser = subparsers.add_parser(
+    break_lock_parser = action_parsers.add_parser(
         'break-lock',
-        aliases=SUBPARSER_ALIASES['break-lock'],
+        aliases=ACTION_ALIASES['break-lock'],
         help='Break the repository and cache locks left behind by Borg aborting',
         description='Break Borg repository and cache locks left behind by Borg aborting',
         add_help=False,
@@ -819,9 +1215,9 @@ def make_parsers():
         '-h', '--help', action='help', help='Show this help message and exit'
     )
 
-    borg_parser = subparsers.add_parser(
+    borg_parser = action_parsers.add_parser(
         'borg',
-        aliases=SUBPARSER_ALIASES['borg'],
+        aliases=ACTION_ALIASES['borg'],
         help='Run an arbitrary Borg command',
         description="Run an arbitrary Borg command based on borgmatic's configuration",
         add_help=False,
@@ -841,20 +1237,41 @@ def make_parsers():
     )
     borg_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
-    return top_level_parser, subparsers
+    return global_parser, action_parsers, global_plus_action_parser
 
 
 def parse_arguments(*unparsed_arguments):
     '''
     Given command-line arguments with which this script was invoked, parse the arguments and return
-    them as a dict mapping from subparser name (or "global") to an argparse.Namespace instance.
-    '''
-    top_level_parser, subparsers = make_parsers()
+    them as a dict mapping from action name (or "global") to an argparse.Namespace instance.
 
-    arguments, remaining_arguments = parse_subparser_arguments(
-        unparsed_arguments, subparsers.choices
+    Raise ValueError if the arguments cannot be parsed.
+    Raise SystemExit with an error code of 0 if "--help" was requested.
+    '''
+    global_parser, action_parsers, global_plus_action_parser = make_parsers()
+    arguments, remaining_action_arguments = parse_arguments_for_actions(
+        unparsed_arguments, action_parsers.choices, global_parser
     )
-    arguments['global'] = top_level_parser.parse_args(remaining_arguments)
+
+    for action_name in ('bootstrap', 'generate', 'validate'):
+        if (
+            action_name in arguments.keys() and len(arguments.keys()) > 2
+        ):  # 2 = 1 for 'global' + 1 for the action
+            raise ValueError(
+                f'The {action_name} action cannot be combined with other actions. Please run it separately.'
+            )
+
+    unknown_arguments = get_unparsable_arguments(remaining_action_arguments)
+
+    if unknown_arguments:
+        if '--help' in unknown_arguments or '-h' in unknown_arguments:
+            global_plus_action_parser.print_help()
+            sys.exit(0)
+
+        global_plus_action_parser.print_usage()
+        raise ValueError(
+            f"Unrecognized argument{'s' if len(unknown_arguments) > 1 else ''}: {' '.join(unknown_arguments)}"
+        )
 
     if arguments['global'].excludes_filename:
         raise ValueError(
@@ -865,6 +1282,10 @@ def parse_arguments(*unparsed_arguments):
         raise ValueError(
             'With the create action, only one of --list (--files) and --progress flags can be used.'
         )
+    if 'create' in arguments and arguments['create'].list_files and arguments['create'].json:
+        raise ValueError(
+            'With the create action, only one of --list (--files) and --json flags can be used.'
+        )
 
     if (
         ('list' in arguments and 'rinfo' in arguments and arguments['list'].json)

borgmatic/commands/borgmatic.py

@@ -8,12 +8,19 @@ from queue import Queue
 from subprocess import CalledProcessError
 
 import colorama
-import pkg_resources
+
+try:
+    import importlib_metadata
+except ModuleNotFoundError:  # pragma: nocover
+    import importlib.metadata as importlib_metadata
 
 import borgmatic.actions.borg
 import borgmatic.actions.break_lock
 import borgmatic.actions.check
 import borgmatic.actions.compact
+import borgmatic.actions.config.bootstrap
+import borgmatic.actions.config.generate
+import borgmatic.actions.config.validate
 import borgmatic.actions.create
 import borgmatic.actions.export_tar
 import borgmatic.actions.extract
@@ -26,13 +33,14 @@ import borgmatic.actions.restore
 import borgmatic.actions.rinfo
 import borgmatic.actions.rlist
 import borgmatic.actions.transfer
-import borgmatic.commands.completion
+import borgmatic.commands.completion.bash
+import borgmatic.commands.completion.fish
 from borgmatic.borg import umount as borg_umount
 from borgmatic.borg import version as borg_version
 from borgmatic.commands.arguments import parse_arguments
-from borgmatic.config import checks, collect, convert, validate
+from borgmatic.config import checks, collect, validate
 from borgmatic.hooks import command, dispatch, monitor
-from borgmatic.logger import add_custom_log_levels, configure_logging, should_do_markup
+from borgmatic.logger import DISABLED, add_custom_log_levels, configure_logging, should_do_markup
 from borgmatic.signals import configure_signals
 from borgmatic.verbosity import verbosity_to_log_level
 
@@ -66,6 +74,7 @@ def run_configuration(config_filename, config, arguments):
     error_repository = ''
     using_primary_action = {'create', 'prune', 'compact', 'check'}.intersection(arguments)
     monitoring_log_level = verbosity_to_log_level(global_arguments.monitoring_verbosity)
+    monitoring_hooks_are_activated = using_primary_action and monitoring_log_level != DISABLED
 
     try:
         local_borg_version = borg_version.local_borg_version(storage, local_path)
@@ -74,7 +83,7 @@ def run_configuration(config_filename, config, arguments):
         return
 
     try:
-        if using_primary_action:
+        if monitoring_hooks_are_activated:
             dispatch.call_hooks(
                 'initialize_monitor',
                 hooks,
@@ -83,7 +92,7 @@ def run_configuration(config_filename, config, arguments):
                 monitoring_log_level,
                 global_arguments.dry_run,
             )
-        if using_primary_action:
+
             dispatch.call_hooks(
                 'ping_monitor',
                 hooks,
@@ -103,14 +112,20 @@ def run_configuration(config_filename, config, arguments):
     if not encountered_error:
         repo_queue = Queue()
         for repo in location['repositories']:
-            repo_queue.put((repo, 0),)
+            repo_queue.put(
+                (repo, 0),
+            )
 
         while not repo_queue.empty():
             repository, retry_num = repo_queue.get()
-            logger.debug(f'{repository["path"]}: Running actions for repository')
+            logger.debug(
+                f'{repository.get("label", repository["path"])}: Running actions for repository'
+            )
             timeout = retry_num * retry_wait
             if timeout:
-                logger.warning(f'{config_filename}: Sleeping {timeout}s before next retry')
+                logger.warning(
+                    f'{repository.get("label", repository["path"])}: Sleeping {timeout}s before next retry'
+                )
                 time.sleep(timeout)
             try:
                 yield from run_actions(
@@ -128,17 +143,19 @@ def run_configuration(config_filename, config, arguments):
                 )
             except (OSError, CalledProcessError, ValueError) as error:
                 if retry_num < retries:
-                    repo_queue.put((repository, retry_num + 1),)
+                    repo_queue.put(
+                        (repository, retry_num + 1),
+                    )
                     tuple(  # Consume the generator so as to trigger logging.
                         log_error_records(
-                            f'{repository["path"]}: Error running actions for repository',
+                            f'{repository.get("label", repository["path"])}: Error running actions for repository',
                             error,
                             levelno=logging.WARNING,
                             log_command_error_output=True,
                         )
                     )
                     logger.warning(
-                        f'{config_filename}: Retrying... attempt {retry_num + 1}/{retries}'
+                        f'{repository.get("label", repository["path"])}: Retrying... attempt {retry_num + 1}/{retries}'
                     )
                     continue
 
@@ -146,13 +163,14 @@ def run_configuration(config_filename, config, arguments):
                     return
 
                 yield from log_error_records(
-                    f'{repository["path"]}: Error running actions for repository', error
+                    f'{repository.get("label", repository["path"])}: Error running actions for repository',
+                    error,
                 )
                 encountered_error = error
                 error_repository = repository['path']
 
     try:
-        if using_primary_action:
+        if monitoring_hooks_are_activated:
             # send logs irrespective of error
             dispatch.call_hooks(
                 'ping_monitor',
@@ -172,7 +190,7 @@ def run_configuration(config_filename, config, arguments):
 
     if not encountered_error:
         try:
-            if using_primary_action:
+            if monitoring_hooks_are_activated:
                 dispatch.call_hooks(
                     'ping_monitor',
                     hooks,
@@ -279,7 +297,7 @@ def run_actions(
         **hook_context,
     )
 
-    for (action_name, action_arguments) in arguments.items():
+    for action_name, action_arguments in arguments.items():
         if action_name == 'rcreate':
             borgmatic.actions.rcreate.run_rcreate(
                 repository,
@@ -390,7 +408,8 @@ def run_actions(
                 repository,
                 storage,
                 local_borg_version,
-                arguments['mount'],
+                action_arguments,
+                global_arguments,
                 local_path,
                 remote_path,
             )
@@ -408,32 +427,63 @@ def run_actions(
             )
         elif action_name == 'rlist':
             yield from borgmatic.actions.rlist.run_rlist(
-                repository, storage, local_borg_version, action_arguments, local_path, remote_path,
+                repository,
+                storage,
+                local_borg_version,
+                action_arguments,
+                global_arguments,
+                local_path,
+                remote_path,
             )
         elif action_name == 'list':
             yield from borgmatic.actions.list.run_list(
-                repository, storage, local_borg_version, action_arguments, local_path, remote_path,
+                repository,
+                storage,
+                local_borg_version,
+                action_arguments,
+                global_arguments,
+                local_path,
+                remote_path,
             )
         elif action_name == 'rinfo':
             yield from borgmatic.actions.rinfo.run_rinfo(
-                repository, storage, local_borg_version, action_arguments, local_path, remote_path,
+                repository,
+                storage,
+                local_borg_version,
+                action_arguments,
+                global_arguments,
+                local_path,
+                remote_path,
             )
         elif action_name == 'info':
             yield from borgmatic.actions.info.run_info(
-                repository, storage, local_borg_version, action_arguments, local_path, remote_path,
+                repository,
+                storage,
+                local_borg_version,
+                action_arguments,
+                global_arguments,
+                local_path,
+                remote_path,
             )
         elif action_name == 'break-lock':
             borgmatic.actions.break_lock.run_break_lock(
                 repository,
                 storage,
                 local_borg_version,
-                arguments['break-lock'],
+                action_arguments,
+                global_arguments,
                 local_path,
                 remote_path,
             )
         elif action_name == 'borg':
             borgmatic.actions.borg.run_borg(
-                repository, storage, local_borg_version, action_arguments, local_path, remote_path,
+                repository,
+                storage,
+                local_borg_version,
+                action_arguments,
+                global_arguments,
+                local_path,
+                remote_path,
             )
 
     command.execute_hook(
@@ -451,6 +501,9 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True):
     Given a sequence of configuration filenames, load and validate each configuration file. Return
     the results as a tuple of: dict of configuration filename to corresponding parsed configuration,
     and sequence of logging.LogRecord instances containing any parse errors.
+
+    Log records are returned here instead of being logged directly because logging isn't yet
+    initialized at this point!
     '''
     # Dict mapping from config filename to corresponding parsed config dict.
     configs = collections.OrderedDict()
@@ -458,6 +511,17 @@ def load_configurations(config_filenames, overrides=None, resolve_env=True):
 
     # Parse and load each configuration file.
     for config_filename in config_filenames:
+        logs.extend(
+            [
+                logging.makeLogRecord(
+                    dict(
+                        levelno=logging.DEBUG,
+                        levelname='DEBUG',
+                        msg=f'{config_filename}: Loading configuration file',
+                    )
+                ),
+            ]
+        )
         try:
             configs[config_filename], parse_logs = validate.parse_configuration(
                 config_filename, validate.schema_filename(), overrides, resolve_env
@@ -554,9 +618,101 @@ def get_local_path(configs):
     return next(iter(configs.values())).get('location', {}).get('local_path', 'borg')
 
 
+def collect_highlander_action_summary_logs(configs, arguments, configuration_parse_errors):
+    '''
+    Given a dict of configuration filename to corresponding parsed configuration, parsed
+    command-line arguments as a dict from subparser name to a parsed namespace of arguments, and
+    whether any configuration files encountered errors during parsing, run a highlander action
+    specified in the arguments, if any, and yield a series of logging.LogRecord instances containing
+    summary information.
+
+    A highlander action is an action that cannot coexist with other actions on the borgmatic
+    command-line, and borgmatic exits after processing such an action.
+    '''
+    if 'bootstrap' in arguments:
+        try:
+            # No configuration file is needed for bootstrap.
+            local_borg_version = borg_version.local_borg_version({}, 'borg')
+        except (OSError, CalledProcessError, ValueError) as error:
+            yield from log_error_records('Error getting local Borg version', error)
+            return
+
+        try:
+            borgmatic.actions.config.bootstrap.run_bootstrap(
+                arguments['bootstrap'], arguments['global'], local_borg_version
+            )
+            yield logging.makeLogRecord(
+                dict(
+                    levelno=logging.ANSWER,
+                    levelname='ANSWER',
+                    msg='Bootstrap successful',
+                )
+            )
+        except (
+            CalledProcessError,
+            ValueError,
+            OSError,
+        ) as error:
+            yield from log_error_records(error)
+
+        return
+
+    if 'generate' in arguments:
+        try:
+            borgmatic.actions.config.generate.run_generate(
+                arguments['generate'], arguments['global']
+            )
+            yield logging.makeLogRecord(
+                dict(
+                    levelno=logging.ANSWER,
+                    levelname='ANSWER',
+                    msg='Generate successful',
+                )
+            )
+        except (
+            CalledProcessError,
+            ValueError,
+            OSError,
+        ) as error:
+            yield from log_error_records(error)
+
+        return
+
+    if 'validate' in arguments:
+        if configuration_parse_errors:
+            yield logging.makeLogRecord(
+                dict(
+                    levelno=logging.CRITICAL,
+                    levelname='CRITICAL',
+                    msg='Configuration validation failed',
+                )
+            )
+
+            return
+
+        try:
+            borgmatic.actions.config.validate.run_validate(arguments['validate'], configs)
+
+            yield logging.makeLogRecord(
+                dict(
+                    levelno=logging.ANSWER,
+                    levelname='ANSWER',
+                    msg='All configuration files are valid',
+                )
+            )
+        except (
+            CalledProcessError,
+            ValueError,
+            OSError,
+        ) as error:
+            yield from log_error_records(error)
+
+        return
+
+
 def collect_configuration_run_summary_logs(configs, arguments):
     '''
-    Given a dict of configuration filename to corresponding parsed configuration, and parsed
+    Given a dict of configuration filename to corresponding parsed configuration and parsed
     command-line arguments as a dict from subparser name to a parsed namespace of arguments, run
     each configuration file and yield a series of logging.LogRecord instances containing summary
     information about each run.
@@ -626,7 +782,8 @@ def collect_configuration_run_summary_logs(configs, arguments):
         logger.info(f"Unmounting mount point {arguments['umount'].mount_point}")
         try:
             borg_umount.unmount_archive(
-                mount_point=arguments['umount'].mount_point, local_path=get_local_path(configs),
+                mount_point=arguments['umount'].mount_point,
+                local_path=get_local_path(configs),
             )
         except (CalledProcessError, OSError) as error:
             yield from log_error_records('Error unmounting mount point', error)
@@ -658,7 +815,7 @@ def exit_with_help_link():  # pragma: no cover
     sys.exit(1)
 
 
-def main():  # pragma: no cover
+def main(extra_summary_logs=[]):  # pragma: no cover
     configure_signals()
 
     try:
@@ -676,16 +833,23 @@ def main():  # pragma: no cover
 
     global_arguments = arguments['global']
     if global_arguments.version:
-        print(pkg_resources.require('borgmatic')[0].version)
+        print(importlib_metadata.version('borgmatic'))
         sys.exit(0)
     if global_arguments.bash_completion:
-        print(borgmatic.commands.completion.bash_completion())
+        print(borgmatic.commands.completion.bash.bash_completion())
+        sys.exit(0)
+    if global_arguments.fish_completion:
+        print(borgmatic.commands.completion.fish.fish_completion())
         sys.exit(0)
 
     config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
+    global_arguments.used_config_paths = list(config_filenames)
     configs, parse_logs = load_configurations(
         config_filenames, global_arguments.overrides, global_arguments.resolve_env
     )
+    configuration_parse_errors = (
+        (max(log.levelno for log in parse_logs) >= logging.CRITICAL) if parse_logs else False
+    )
 
     any_json_flags = any(
         getattr(sub_arguments, 'json', False) for sub_arguments in arguments.values()
@@ -708,10 +872,18 @@ def main():  # pragma: no cover
         logger.critical(f'Error configuring logging: {error}')
         exit_with_help_link()
 
-    logger.debug('Ensuring legacy configuration is upgraded')
-    convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH, config_filenames)
-
-    summary_logs = parse_logs + list(collect_configuration_run_summary_logs(configs, arguments))
+    summary_logs = (
+        extra_summary_logs
+        + parse_logs
+        + (
+            list(
+                collect_highlander_action_summary_logs(
+                    configs, arguments, configuration_parse_errors
+                )
+            )
+            or list(collect_configuration_run_summary_logs(configs, arguments))
+        )
+    )
     summary_logs_max_level = max(log.levelno for log in summary_logs)
 
     for message in ('', 'summary:'):

borgmatic/commands/completion/actions.py

@@ -0,0 +1,36 @@
+import borgmatic.commands.arguments
+
+
+def upgrade_message(language: str, upgrade_command: str, completion_file: str):
+    return f'''
+Your {language} completions script is from a different version of borgmatic than is
+currently installed. Please upgrade your script so your completions match the
+command-line flags in your installed borgmatic! Try this to upgrade:
+
+    {upgrade_command}
+    source {completion_file}
+'''
+
+
+def available_actions(subparsers, current_action=None):
+    '''
+    Given subparsers as an argparse._SubParsersAction instance and a current action name (if
+    any), return the actions names that can follow the current action on a command-line.
+
+    This takes into account which sub-actions that the current action supports. For instance, if
+    "bootstrap" is a sub-action for "config", then "bootstrap" should be able to follow a current
+    action of "config" but not "list".
+    '''
+    action_to_subactions = borgmatic.commands.arguments.get_subactions_for_actions(
+        subparsers.choices
+    )
+    current_subactions = action_to_subactions.get(current_action)
+
+    if current_subactions:
+        return current_subactions
+
+    all_subactions = set(
+        subaction for subactions in action_to_subactions.values() for subaction in subactions
+    )
+
+    return tuple(action for action in subparsers.choices.keys() if action not in all_subactions)

borgmatic/commands/completion/bash.py

@@ -1,13 +1,5 @@
-from borgmatic.commands import arguments
-
-UPGRADE_MESSAGE = '''
-Your bash completions script is from a different version of borgmatic than is
-currently installed. Please upgrade your script so your completions match the
-command-line flags in your installed borgmatic! Try this to upgrade:
-
-    sudo sh -c "borgmatic --bash-completion > $BASH_SOURCE"
-    source $BASH_SOURCE
-'''
+import borgmatic.commands.arguments
+import borgmatic.commands.completion.actions
 
 
 def parser_flags(parser):
@@ -23,9 +15,12 @@ def bash_completion():
     Return a bash completion script for the borgmatic command. Produce this by introspecting
     borgmatic's command-line argument parsers.
     '''
-    top_level_parser, subparsers = arguments.make_parsers()
-    global_flags = parser_flags(top_level_parser)
-    actions = ' '.join(subparsers.choices.keys())
+    (
+        unused_global_parser,
+        action_parsers,
+        global_plus_action_parser,
+    ) = borgmatic.commands.arguments.make_parsers()
+    global_flags = parser_flags(global_plus_action_parser)
 
     # Avert your eyes.
     return '\n'.join(
@@ -34,7 +29,11 @@ def bash_completion():
             '    local this_script="$(cat "$BASH_SOURCE" 2> /dev/null)"',
             '    local installed_script="$(borgmatic --bash-completion 2> /dev/null)"',
             '    if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ];'
-            f'        then cat << EOF\n{UPGRADE_MESSAGE}\nEOF',
+            f'''        then cat << EOF\n{borgmatic.commands.completion.actions.upgrade_message(
+                    'bash',
+                    'sudo sh -c "borgmatic --bash-completion > $BASH_SOURCE"',
+                    '$BASH_SOURCE',
+                )}\nEOF''',
             '    fi',
             '}',
             'complete_borgmatic() {',
@@ -44,12 +43,22 @@ def bash_completion():
         COMPREPLY=($(compgen -W "%s %s %s" -- "${COMP_WORDS[COMP_CWORD]}"))
         return 0
     fi'''
-            % (action, parser_flags(subparser), actions, global_flags)
-            for action, subparser in subparsers.choices.items()
+            % (
+                action,
+                parser_flags(action_parser),
+                ' '.join(
+                    borgmatic.commands.completion.actions.available_actions(action_parsers, action)
+                ),
+                global_flags,
+            )
+            for action, action_parser in reversed(action_parsers.choices.items())
         )
         + (
             '    COMPREPLY=($(compgen -W "%s %s" -- "${COMP_WORDS[COMP_CWORD]}"))'  # noqa: FS003
-            % (actions, global_flags),
+            % (
+                ' '.join(borgmatic.commands.completion.actions.available_actions(action_parsers)),
+                global_flags,
+            ),
             '    (check_version &)',
             '}',
             '\ncomplete -o bashdefault -o default -F complete_borgmatic borgmatic',

borgmatic/commands/completion/fish.py

@@ -0,0 +1,176 @@
+import shlex
+from argparse import Action
+from textwrap import dedent
+
+import borgmatic.commands.arguments
+import borgmatic.commands.completion.actions
+
+
+def has_file_options(action: Action):
+    '''
+    Given an argparse.Action instance, return True if it takes a file argument.
+    '''
+    return action.metavar in (
+        'FILENAME',
+        'PATH',
+    ) or action.dest in ('config_paths',)
+
+
+def has_choice_options(action: Action):
+    '''
+    Given an argparse.Action instance, return True if it takes one of a predefined set of arguments.
+    '''
+    return action.choices is not None
+
+
+def has_unknown_required_param_options(action: Action):
+    '''
+    A catch-all for options that take a required parameter, but we don't know what the parameter is.
+    This should be used last. These are actions that take something like a glob, a list of numbers, or a string.
+
+    Actions that match this pattern should not show the normal arguments, because those are unlikely to be valid.
+    '''
+    return (
+        action.required is True
+        or action.nargs
+        in (
+            '+',
+            '*',
+        )
+        or action.metavar in ('PATTERN', 'KEYS', 'N')
+        or (action.type is not None and action.default is None)
+    )
+
+
+def has_exact_options(action: Action):
+    return (
+        has_file_options(action)
+        or has_choice_options(action)
+        or has_unknown_required_param_options(action)
+    )
+
+
+def exact_options_completion(action: Action):
+    '''
+    Given an argparse.Action instance, return a completion invocation that forces file completions, options completion,
+    or just that some value follow the action, if the action takes such an argument and was the last action on the
+    command line prior to the cursor.
+
+    Otherwise, return an empty string.
+    '''
+
+    if not has_exact_options(action):
+        return ''
+
+    args = ' '.join(action.option_strings)
+
+    if has_file_options(action):
+        return f'''\ncomplete -c borgmatic -Fr -n "__borgmatic_current_arg {args}"'''
+
+    if has_choice_options(action):
+        return f'''\ncomplete -c borgmatic -f -a '{' '.join(map(str, action.choices))}' -n "__borgmatic_current_arg {args}"'''
+
+    if has_unknown_required_param_options(action):
+        return f'''\ncomplete -c borgmatic -x -n "__borgmatic_current_arg {args}"'''
+
+    raise ValueError(
+        f'Unexpected action: {action} passes has_exact_options but has no choices produced'
+    )
+
+
+def dedent_strip_as_tuple(string: str):
+    '''
+    Dedent a string, then strip it to avoid requiring your first line to have content, then return a tuple of the string.
+    Makes it easier to write multiline strings for completions when you join them with a tuple.
+    '''
+    return (dedent(string).strip('\n'),)
+
+
+def fish_completion():
+    '''
+    Return a fish completion script for the borgmatic command. Produce this by introspecting
+    borgmatic's command-line argument parsers.
+    '''
+    (
+        unused_global_parser,
+        action_parsers,
+        global_plus_action_parser,
+    ) = borgmatic.commands.arguments.make_parsers()
+
+    all_action_parsers = ' '.join(action for action in action_parsers.choices.keys())
+
+    exact_option_args = tuple(
+        ' '.join(action.option_strings)
+        for action_parser in action_parsers.choices.values()
+        for action in action_parser._actions
+        if has_exact_options(action)
+    ) + tuple(
+        ' '.join(action.option_strings)
+        for action in global_plus_action_parser._actions
+        if len(action.option_strings) > 0
+        if has_exact_options(action)
+    )
+
+    # Avert your eyes.
+    return '\n'.join(
+        dedent_strip_as_tuple(
+            f'''
+            function __borgmatic_check_version
+                set -fx this_filename (status current-filename)
+                fish -c '
+                    if test -f "$this_filename"
+                        set this_script (cat $this_filename 2> /dev/null)
+                        set installed_script (borgmatic --fish-completion 2> /dev/null)
+                        if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ]
+                            echo "{borgmatic.commands.completion.actions.upgrade_message(
+                            'fish',
+                            'borgmatic --fish-completion | sudo tee $this_filename',
+                            '$this_filename',
+                        )}"
+                        end
+                    end
+                ' &
+            end
+            __borgmatic_check_version
+
+            function __borgmatic_current_arg --description 'Check if any of the given arguments are the last on the command line before the cursor'
+                set -l all_args (commandline -poc)
+                # premature optimization to avoid iterating all args if there aren't enough
+                # to have a last arg beyond borgmatic
+                if [ (count $all_args) -lt 2 ]
+                    return 1
+                end
+                for arg in $argv
+                    if [ "$arg" = "$all_args[-1]" ]
+                        return 0
+                    end
+                end
+                return 1
+            end
+
+            set --local action_parser_condition "not __fish_seen_subcommand_from {all_action_parsers}"
+            set --local exact_option_condition "not __borgmatic_current_arg {' '.join(exact_option_args)}"
+            '''
+        )
+        + ('\n# action_parser completions',)
+        + tuple(
+            f'''complete -c borgmatic -f -n "$action_parser_condition" -n "$exact_option_condition" -a '{action_name}' -d {shlex.quote(action_parser.description)}'''
+            for action_name, action_parser in action_parsers.choices.items()
+        )
+        + ('\n# global flags',)
+        + tuple(
+            # -n is checked in order, so put faster / more likely to be true checks first
+            f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{exact_options_completion(action)}'''
+            for action in global_plus_action_parser._actions
+            # ignore the noargs action, as this is an impossible completion for fish
+            if len(action.option_strings) > 0
+            if 'Deprecated' not in action.help
+        )
+        + ('\n# action_parser flags',)
+        + tuple(
+            f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}"{exact_options_completion(action)}'''
+            for action_name, action_parser in action_parsers.choices.items()
+            for action in action_parser._actions
+            if 'Deprecated' not in (action.help or ())
+        )
+    )

borgmatic/commands/convert_config.py

@@ -1,102 +0,0 @@
-import os
-import sys
-import textwrap
-from argparse import ArgumentParser
-
-from ruamel import yaml
-
-from borgmatic.config import convert, generate, legacy, validate
-
-DEFAULT_SOURCE_CONFIG_FILENAME = '/etc/borgmatic/config'
-DEFAULT_SOURCE_EXCLUDES_FILENAME = '/etc/borgmatic/excludes'
-DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
-
-
-def parse_arguments(*arguments):
-    '''
-    Given command-line arguments with which this script was invoked, parse the arguments and return
-    them as an ArgumentParser instance.
-    '''
-    parser = ArgumentParser(
-        description='''
-            Convert legacy INI-style borgmatic configuration and excludes files to a single YAML
-            configuration file. Note that this replaces any comments from the source files.
-        '''
-    )
-    parser.add_argument(
-        '-s',
-        '--source-config',
-        dest='source_config_filename',
-        default=DEFAULT_SOURCE_CONFIG_FILENAME,
-        help=f'Source INI-style configuration filename. Default: {DEFAULT_SOURCE_CONFIG_FILENAME}',
-    )
-    parser.add_argument(
-        '-e',
-        '--source-excludes',
-        dest='source_excludes_filename',
-        default=DEFAULT_SOURCE_EXCLUDES_FILENAME
-        if os.path.exists(DEFAULT_SOURCE_EXCLUDES_FILENAME)
-        else None,
-        help='Excludes filename',
-    )
-    parser.add_argument(
-        '-d',
-        '--destination-config',
-        dest='destination_config_filename',
-        default=DEFAULT_DESTINATION_CONFIG_FILENAME,
-        help=f'Destination YAML configuration filename. Default: {DEFAULT_DESTINATION_CONFIG_FILENAME}',
-    )
-
-    return parser.parse_args(arguments)
-
-
-TEXT_WRAP_CHARACTERS = 80
-
-
-def display_result(args):  # pragma: no cover
-    result_lines = textwrap.wrap(
-        f'Your borgmatic configuration has been upgraded. Please review the result in {args.destination_config_filename}.',
-        TEXT_WRAP_CHARACTERS,
-    )
-
-    excludes_phrase = (
-        f' and {args.source_excludes_filename}' if args.source_excludes_filename else ''
-    )
-    delete_lines = textwrap.wrap(
-        f'Once you are satisfied, you can safely delete {args.source_config_filename}{excludes_phrase}.',
-        TEXT_WRAP_CHARACTERS,
-    )
-
-    print('\n'.join(result_lines))
-    print()
-    print('\n'.join(delete_lines))
-
-
-def main():  # pragma: no cover
-    try:
-        args = parse_arguments(*sys.argv[1:])
-        schema = yaml.round_trip_load(open(validate.schema_filename()).read())
-        source_config = legacy.parse_configuration(
-            args.source_config_filename, legacy.CONFIG_FORMAT
-        )
-        source_config_file_mode = os.stat(args.source_config_filename).st_mode
-        source_excludes = (
-            open(args.source_excludes_filename).read().splitlines()
-            if args.source_excludes_filename
-            else []
-        )
-
-        destination_config = convert.convert_legacy_parsed_config(
-            source_config, source_excludes, schema
-        )
-
-        generate.write_configuration(
-            args.destination_config_filename,
-            generate.render_configuration(destination_config),
-            mode=source_config_file_mode,
-        )
-
-        display_result(args)
-    except (ValueError, OSError) as error:
-        print(error, file=sys.stderr)
-        sys.exit(1)

borgmatic/commands/generate_config.py

@@ -1,63 +1,17 @@
+import logging
 import sys
-from argparse import ArgumentParser
 
-from borgmatic.config import generate, validate
-
-DEFAULT_DESTINATION_CONFIG_FILENAME = '/etc/borgmatic/config.yaml'
+import borgmatic.commands.borgmatic
 
 
-def parse_arguments(*arguments):
-    '''
-    Given command-line arguments with which this script was invoked, parse the arguments and return
-    them as an ArgumentParser instance.
-    '''
-    parser = ArgumentParser(description='Generate a sample borgmatic YAML configuration file.')
-    parser.add_argument(
-        '-s',
-        '--source',
-        dest='source_filename',
-        help='Optional YAML configuration file to merge into the generated configuration, useful for upgrading your configuration',
-    )
-    parser.add_argument(
-        '-d',
-        '--destination',
-        dest='destination_filename',
-        default=DEFAULT_DESTINATION_CONFIG_FILENAME,
-        help=f'Destination YAML configuration file, default: {DEFAULT_DESTINATION_CONFIG_FILENAME}',
-    )
-    parser.add_argument(
-        '--overwrite',
-        default=False,
-        action='store_true',
-        help='Whether to overwrite any existing destination file, defaults to false',
-    )
-
-    return parser.parse_args(arguments)
-
-
-def main():  # pragma: no cover
-    try:
-        args = parse_arguments(*sys.argv[1:])
-
-        generate.generate_sample_configuration(
-            args.source_filename,
-            args.destination_filename,
-            validate.schema_filename(),
-            overwrite=args.overwrite,
+def main():
+    warning_log = logging.makeLogRecord(
+        dict(
+            levelno=logging.WARNING,
+            levelname='WARNING',
+            msg='generate-borgmatic-config is deprecated and will be removed from a future release. Please use "borgmatic config generate" instead.',
         )
+    )
 
-        print(f'Generated a sample configuration file at {args.destination_filename}.')
-        print()
-        if args.source_filename:
-            print(f'Merged in the contents of configuration file at {args.source_filename}.')
-            print('To review the changes made, run:')
-            print()
-            print(f'    diff --unified {args.source_filename} {args.destination_filename}')
-            print()
-        print('This includes all available configuration options with example values. The few')
-        print('required options are indicated. Please edit the file to suit your needs.')
-        print()
-        print('If you ever need help: https://torsion.org/borgmatic/#issues')
-    except (ValueError, OSError) as error:
-        print(error, file=sys.stderr)
-        sys.exit(1)
+    sys.argv = ['borgmatic', 'config', 'generate'] + sys.argv[1:]
+    borgmatic.commands.borgmatic.main([warning_log])

borgmatic/commands/validate_config.py

@@ -1,68 +1,17 @@
 import logging
 import sys
-from argparse import ArgumentParser
 
-import borgmatic.config.generate
-from borgmatic.config import collect, validate
-
-logger = logging.getLogger(__name__)
+import borgmatic.commands.borgmatic
 
 
-def parse_arguments(*arguments):
-    '''
-    Given command-line arguments with which this script was invoked, parse the arguments and return
-    them as an ArgumentParser instance.
-    '''
-    config_paths = collect.get_default_config_paths()
-
-    parser = ArgumentParser(description='Validate borgmatic configuration file(s).')
-    parser.add_argument(
-        '-c',
-        '--config',
-        nargs='+',
-        dest='config_paths',
-        default=config_paths,
-        help=f'Configuration filenames or directories, defaults to: {config_paths}',
-    )
-    parser.add_argument(
-        '-s',
-        '--show',
-        action='store_true',
-        help='Show the validated configuration after all include merging has occurred',
+def main():
+    warning_log = logging.makeLogRecord(
+        dict(
+            levelno=logging.WARNING,
+            levelname='WARNING',
+            msg='validate-borgmatic-config is deprecated and will be removed from a future release. Please use "borgmatic config validate" instead.',
+        )
     )
 
-    return parser.parse_args(arguments)
-
-
-def main():  # pragma: no cover
-    arguments = parse_arguments(*sys.argv[1:])
-
-    logging.basicConfig(level=logging.INFO, format='%(message)s')
-
-    config_filenames = tuple(collect.collect_config_filenames(arguments.config_paths))
-    if len(config_filenames) == 0:
-        logger.critical('No files to validate found')
-        sys.exit(1)
-
-    found_issues = False
-    for config_filename in config_filenames:
-        try:
-            config, parse_logs = validate.parse_configuration(
-                config_filename, validate.schema_filename()
-            )
-        except (ValueError, OSError, validate.Validation_error) as error:
-            logging.critical(f'{config_filename}: Error parsing configuration file')
-            logging.critical(error)
-            found_issues = True
-        else:
-            for log in parse_logs:
-                logger.handle(log)
-
-            if arguments.show:
-                print('---')
-                print(borgmatic.config.generate.render_configuration(config))
-
-    if found_issues:
-        sys.exit(1)
-
-    logger.info(f"All given configuration files are valid: {', '.join(config_filenames)}")
+    sys.argv = ['borgmatic', 'config', 'validate'] + sys.argv[1:]
+    borgmatic.commands.borgmatic.main([warning_log])

borgmatic/config/collect.py

@@ -24,9 +24,9 @@ def get_default_config_paths(expand_home=True):
 def collect_config_filenames(config_paths):
     '''
     Given a sequence of config paths, both filenames and directories, resolve that to an iterable
-    of files. Accomplish this by listing any given directories looking for contained config files
-    (ending with the ".yaml" or ".yml" extension). This is non-recursive, so any directories within the given
-    directories are ignored.
+    of absolute files. Accomplish this by listing any given directories looking for contained config
+    files (ending with the ".yaml" or ".yml" extension). This is non-recursive, so any directories
+    within the given directories are ignored.
 
     Return paths even if they don't exist on disk, so the user can find out about missing
     configuration paths. However, skip a default config path if it's missing, so the user doesn't
@@ -41,7 +41,7 @@ def collect_config_filenames(config_paths):
             continue
 
         if not os.path.isdir(path) or not exists:
-            yield path
+            yield os.path.abspath(path)
             continue
 
         if not os.access(path, os.R_OK):
@@ -51,4 +51,4 @@ def collect_config_filenames(config_paths):
             full_filename = os.path.join(path, filename)
             matching_filetype = full_filename.endswith('.yaml') or full_filename.endswith('.yml')
             if matching_filetype and not os.path.isdir(full_filename):
-                yield full_filename
+                yield os.path.abspath(full_filename)

borgmatic/config/convert.py

@@ -1,95 +0,0 @@
-import os
-
-from ruamel import yaml
-
-from borgmatic.config import generate
-
-
-def _convert_section(source_section_config, section_schema):
-    '''
-    Given a legacy Parsed_config instance for a single section, convert it to its corresponding
-    yaml.comments.CommentedMap representation in preparation for actual serialization to YAML.
-
-    Where integer types exist in the given section schema, convert their values to integers.
-    '''
-    destination_section_config = yaml.comments.CommentedMap(
-        [
-            (
-                option_name,
-                int(option_value)
-                if section_schema['properties'].get(option_name, {}).get('type') == 'integer'
-                else option_value,
-            )
-            for option_name, option_value in source_section_config.items()
-        ]
-    )
-
-    return destination_section_config
-
-
-def convert_legacy_parsed_config(source_config, source_excludes, schema):
-    '''
-    Given a legacy Parsed_config instance loaded from an INI-style config file and a list of exclude
-    patterns, convert them to a corresponding yaml.comments.CommentedMap representation in
-    preparation for serialization to a single YAML config file.
-
-    Additionally, use the given schema as a source of helpful comments to include within the
-    returned CommentedMap.
-    '''
-    destination_config = yaml.comments.CommentedMap(
-        [
-            (section_name, _convert_section(section_config, schema['properties'][section_name]))
-            for section_name, section_config in source_config._asdict().items()
-        ]
-    )
-
-    # Split space-separated values into actual lists, make "repository" into a list, and merge in
-    # excludes.
-    location = destination_config['location']
-    location['source_directories'] = source_config.location['source_directories'].split(' ')
-    location['repositories'] = [location.pop('repository')]
-    location['exclude_patterns'] = source_excludes
-
-    if source_config.consistency.get('checks'):
-        destination_config['consistency']['checks'] = source_config.consistency['checks'].split(' ')
-
-    # Add comments to each section, and then add comments to the fields in each section.
-    generate.add_comments_to_configuration_object(destination_config, schema)
-
-    for section_name, section_config in destination_config.items():
-        generate.add_comments_to_configuration_object(
-            section_config, schema['properties'][section_name], indent=generate.INDENT
-        )
-
-    return destination_config
-
-
-class Legacy_configuration_not_upgraded(FileNotFoundError):
-    def __init__(self):
-        super(Legacy_configuration_not_upgraded, self).__init__(
-            '''borgmatic changed its configuration file format in version 1.1.0 from INI-style
-to YAML. This better supports validation, and has a more natural way to express
-lists of values. To upgrade your existing configuration, run:
-
-    sudo upgrade-borgmatic-config
-
-That will generate a new YAML configuration file at /etc/borgmatic/config.yaml
-(by default) using the values from both your existing configuration and excludes
-files. The new version of borgmatic will consume the YAML configuration file
-instead of the old one.'''
-        )
-
-
-def guard_configuration_upgraded(source_config_filename, destination_config_filenames):
-    '''
-    If legacy source configuration exists but no destination upgraded configs do, raise
-    Legacy_configuration_not_upgraded.
-
-    The idea is that we want to alert the user about upgrading their config if they haven't already.
-    '''
-    destination_config_exists = any(
-        os.path.exists(filename) for filename in destination_config_filenames
-    )
-
-    if os.path.exists(source_config_filename) and not destination_config_exists:
-        raise Legacy_configuration_not_upgraded()

borgmatic/config/generate.py

@@ -260,14 +260,14 @@ def merge_source_configuration_into_destination(destination_config, source_confi
             )
             continue
 
-        # This is some sort of scalar. Simply set it into the destination.
+        # This is some sort of scalar. Set it into the destination.
         destination_config[field_name] = source_config[field_name]
 
     return destination_config
 
 
 def generate_sample_configuration(
-    source_filename, destination_filename, schema_filename, overwrite=False
+    dry_run, source_filename, destination_filename, schema_filename, overwrite=False
 ):
     '''
     Given an optional source configuration filename, and a required destination configuration
@@ -287,6 +287,9 @@ def generate_sample_configuration(
         _schema_to_sample_configuration(schema), source_config
     )
 
+    if dry_run:
+        return
+
     write_configuration(
         destination_filename,
         _comment_out_optional_configuration(render_configuration(destination_config)),

borgmatic/config/load.py

@@ -121,6 +121,7 @@ def load_configuration(filename):
     Raise ruamel.yaml.error.YAMLError if something goes wrong parsing the YAML, or RecursionError
     if there are too many recursive includes.
     '''
+
     # Use an embedded derived class for the include constructor so as to capture the filename
     # value. (functools.partial doesn't work for this use case because yaml.Constructor has to be
     # an actual class.)

borgmatic/config/normalize.py

@@ -81,7 +81,10 @@ def normalize(config_filename, config):
                         repository_path.partition('file://')[-1]
                     )
                     config['location']['repositories'].append(
-                        dict(repository_dict, path=updated_repository_path,)
+                        dict(
+                            repository_dict,
+                            path=updated_repository_path,
+                        )
                     )
                 elif repository_path.startswith('ssh://'):
                     config['location']['repositories'].append(repository_dict)
@@ -97,7 +100,10 @@ def normalize(config_filename, config):
                         )
                     )
                     config['location']['repositories'].append(
-                        dict(repository_dict, path=rewritten_repository_path,)
+                        dict(
+                            repository_dict,
+                            path=rewritten_repository_path,
+                        )
                     )
             else:
                 config['location']['repositories'].append(repository_dict)

borgmatic/config/override.py

@@ -57,7 +57,12 @@ def parse_overrides(raw_overrides):
     for raw_override in raw_overrides:
         try:
             raw_keys, value = raw_override.split('=', 1)
-            parsed_overrides.append((tuple(raw_keys.split('.')), convert_value_type(value),))
+            parsed_overrides.append(
+                (
+                    tuple(raw_keys.split('.')),
+                    convert_value_type(value),
+                )
+            )
         except ValueError:
             raise ValueError(
                 f"Invalid override '{raw_override}'. Make sure you use the form: SECTION.OPTION=VALUE"
@@ -75,5 +80,5 @@ def apply_overrides(config, raw_overrides):
     '''
     overrides = parse_overrides(raw_overrides)
 
-    for (keys, value) in overrides:
+    for keys, value in overrides:
         set_values(config, keys, value)

borgmatic/config/schema.yaml

@@ -30,8 +30,8 @@ properties:
                 items:
                     type: string
                 description: |
-                    List of source directories to backup. Globs and tildes are
-                    expanded. Do not backslash spaces in path names.
+                    List of source directories and files to backup. Globs and
+                    tildes are expanded. Do not backslash spaces in path names.
                 example:
                     - /home
                     - /etc
@@ -60,7 +60,7 @@ properties:
                     or port. If systemd service is used, then add local
                     repository paths in the systemd service file to the
                     ReadWritePaths list. Prior to borgmatic 1.7.10, repositories
-                    was just a list of plain path strings.
+                    was a list of plain path strings.
                 example:
                     - path: ssh://user@backupserver/./sourcehostname.borg
                       label: backupserver
@@ -605,7 +605,7 @@ properties:
                     List of one or more shell commands or scripts to execute
                     before all the actions for each repository.
                 example:
-                    - echo "Starting actions."
+                    - "echo Starting actions."
             before_backup:
                 type: array
                 items:
@@ -614,7 +614,7 @@ properties:
                     List of one or more shell commands or scripts to execute
                     before creating a backup, run once per repository.
                 example:
-                    - echo "Starting a backup."
+                    - "echo Starting a backup."
             before_prune:
                 type: array
                 items:
@@ -623,7 +623,7 @@ properties:
                     List of one or more shell commands or scripts to execute
                     before pruning, run once per repository.
                 example:
-                    - echo "Starting pruning."
+                    - "echo Starting pruning."
             before_compact:
                 type: array
                 items:
@@ -632,7 +632,7 @@ properties:
                     List of one or more shell commands or scripts to execute
                     before compaction, run once per repository.
                 example:
-                    - echo "Starting compaction."
+                    - "echo Starting compaction."
             before_check:
                 type: array
                 items:
@@ -641,7 +641,7 @@ properties:
                     List of one or more shell commands or scripts to execute
                     before consistency checks, run once per repository.
                 example:
-                    - echo "Starting checks."
+                    - "echo Starting checks."
             before_extract:
                 type: array
                 items:
@@ -650,7 +650,7 @@ properties:
                     List of one or more shell commands or scripts to execute
                     before extracting a backup, run once per repository.
                 example:
-                    - echo "Starting extracting."
+                    - "echo Starting extracting."
             after_backup:
                 type: array
                 items:
@@ -659,7 +659,7 @@ properties:
                     List of one or more shell commands or scripts to execute
                     after creating a backup, run once per repository.
                 example:
-                    - echo "Finished a backup."
+                    - "echo Finished a backup."
             after_compact:
                 type: array
                 items:
@@ -668,7 +668,7 @@ properties:
                     List of one or more shell commands or scripts to execute
                     after compaction, run once per repository.
                 example:
-                    - echo "Finished compaction."
+                    - "echo Finished compaction."
             after_prune:
                 type: array
                 items:
@@ -677,7 +677,7 @@ properties:
                     List of one or more shell commands or scripts to execute
                     after pruning, run once per repository.
                 example:
-                    - echo "Finished pruning."
+                    - "echo Finished pruning."
             after_check:
                 type: array
                 items:
@@ -686,7 +686,7 @@ properties:
                     List of one or more shell commands or scripts to execute
                     after consistency checks, run once per repository.
                 example:
-                    - echo "Finished checks."
+                    - "echo Finished checks."
             after_extract:
                 type: array
                 items:
@@ -695,7 +695,7 @@ properties:
                     List of one or more shell commands or scripts to execute
                     after extracting a backup, run once per repository.
                 example:
-                    - echo "Finished extracting."
+                    - "echo Finished extracting."
             after_actions:
                 type: array
                 items:
@@ -704,7 +704,7 @@ properties:
                     List of one or more shell commands or scripts to execute
                     after all actions for each repository.
                 example:
-                    - echo "Finished actions."
+                    - "echo Finished actions."
             on_error:
                 type: array
                 items:
@@ -715,7 +715,7 @@ properties:
                     "compact", or "check" action or an associated before/after
                     hook.
                 example:
-                    - echo "Error during create/prune/compact/check."
+                    - "echo Error during create/prune/compact/check."
             before_everything:
                 type: array
                 items:
@@ -726,7 +726,7 @@ properties:
                     These are collected from all configuration files and then
                     run once before all of them (prior to all actions).
                 example:
-                    - echo "Starting actions."
+                    - "echo Starting actions."
             after_everything:
                 type: array
                 items:
@@ -737,7 +737,7 @@ properties:
                     These are collected from all configuration files and then
                     run once after all of them (after any action).
                 example:
-                    - echo "Completed actions."
+                    - "echo Completed actions."
             postgresql_databases:
                 type: array
                 items:
@@ -763,10 +763,21 @@ properties:
                                 Database hostname to connect to. Defaults to
                                 connecting via local Unix socket.
                             example: database.example.org
+                        restore_hostname:
+                            type: string
+                            description: |
+                                Database hostname to restore to. Defaults to
+                                the "hostname" option.
+                            example: database.example.org
                         port:
                             type: integer
                             description: Port to connect to. Defaults to 5432.
                             example: 5433
+                        restore_port:
+                            type: integer
+                            description: Port to restore to. Defaults to the
+                                "port" option.
+                            example: 5433
                         username:
                             type: string
                             description: |
@@ -775,6 +786,12 @@ properties:
                                 You probably want to specify the "postgres"
                                 superuser here when the database name is "all".
                             example: dbuser
+                        restore_username:
+                            type: string
+                            description: |
+                                Username with which to restore the database.
+                                Defaults to the "username" option.
+                            example: dbuser
                         password:
                             type: string
                             description: |
@@ -784,6 +801,24 @@ properties:
                                 without a password or you create a ~/.pgpass
                                 file.
                             example: trustsome1
+                        restore_password:
+                            type: string
+                            description: |
+                                Password with which to connect to the restore
+                                database. Defaults to the "password" option.
+                            example: trustsome1
+                        no_owner:
+                            type: boolean
+                            description: |
+                                Do not output commands to set ownership of
+                                objects to match the original database. By
+                                default, pg_dump and pg_restore issue ALTER 
+                                OWNER or SET SESSION AUTHORIZATION statements 
+                                to set ownership of created schema elements. 
+                                These statements will fail unless the initial
+                                connection to the database is made by a 
+                                superuser.
+                            example: true
                         format:
                             type: string
                             enum: ['plain', 'custom', 'directory', 'tar']
@@ -836,25 +871,25 @@ properties:
                                 Command to use instead of "pg_dump" or
                                 "pg_dumpall". This can be used to run a specific
                                 pg_dump version (e.g., one inside a running
-                                docker container). Defaults to "pg_dump" for
-                                single database dump or "pg_dumpall" to dump
-                                all databases.
+                                container). Defaults to "pg_dump" for single
+                                database dump or "pg_dumpall" to dump all
+                                databases.
                             example: docker exec my_pg_container pg_dump
                         pg_restore_command:
                             type: string
                             description: |
                                 Command to use instead of "pg_restore". This
                                 can be used to run a specific pg_restore
-                                version (e.g., one inside a running docker
-                                container). Defaults to "pg_restore".
+                                version (e.g., one inside a running container).
+                                Defaults to "pg_restore".
                             example: docker exec my_pg_container pg_restore
                         psql_command:
                             type: string
                             description: |
                                 Command to use instead of "psql". This can be
                                 used to run a specific psql version (e.g.,
-                                one inside a running docker container).
-                                Defaults to "psql".
+                                one inside a running container). Defaults to
+                                "psql".
                             example: docker exec my_pg_container psql
                         options:
                             type: string
@@ -919,16 +954,33 @@ properties:
                                 Database hostname to connect to. Defaults to
                                 connecting via local Unix socket.
                             example: database.example.org
+                        restore_hostname:
+                            type: string
+                            description: |
+                                Database hostname to restore to. Defaults to
+                                the "hostname" option.
+                            example: database.example.org
                         port:
                             type: integer
                             description: Port to connect to. Defaults to 3306.
                             example: 3307
+                        restore_port:
+                            type: integer
+                            description: Port to restore to. Defaults to the
+                                "port" option.
+                            example: 5433
                         username:
                             type: string
                             description: |
                                 Username with which to connect to the database.
                                 Defaults to the username of the current user.
                             example: dbuser
+                        restore_username:
+                            type: string
+                            description: |
+                                Username with which to restore the database.
+                                Defaults to the "username" option.
+                            example: dbuser                        
                         password:
                             type: string
                             description: |
@@ -937,6 +989,12 @@ properties:
                                 configured to trust the configured username
                                 without a password.
                             example: trustsome1
+                        restore_password:
+                            type: string
+                            description: |
+                                Password with which to connect to the restore
+                                database. Defaults to the "password" option.
+                            example: trustsome1                        
                         format:
                             type: string
                             enum: ['sql']
@@ -1014,6 +1072,12 @@ properties:
                                 read_special and one_file_system (see above) to
                                 support dump and restore streaming.
                             example: /var/lib/sqlite/users.db
+                        restore_path:
+                            type: string
+                            description: |
+                                Path to the SQLite database file to restore to.
+                                Defaults to the "path" option.
+                            example: /var/lib/sqlite/users.db
             mongodb_databases:
                 type: array
                 items:
@@ -1036,22 +1100,45 @@ properties:
                                 Database hostname to connect to. Defaults to
                                 connecting to localhost.
                             example: database.example.org
+                        restore_hostname:
+                            type: string
+                            description: |
+                                Database hostname to restore to. Defaults to
+                                the "hostname" option.
+                            example: database.example.org
                         port:
                             type: integer
                             description: Port to connect to. Defaults to 27017.
                             example: 27018
+                        restore_port:
+                            type: integer
+                            description: Port to restore to. Defaults to the
+                                "port" option.
+                            example: 5433                        
                         username:
                             type: string
                             description: |
                                 Username with which to connect to the database.
                                 Skip it if no authentication is needed.
                             example: dbuser
+                        restore_username:
+                            type: string
+                            description: |
+                                Username with which to restore the database.
+                                Defaults to the "username" option.
+                            example: dbuser                        
                         password:
                             type: string
                             description: |
                                 Password with which to connect to the database.
                                 Skip it if no authentication is needed.
                             example: trustsome1
+                        restore_password:
+                            type: string
+                            description: |
+                                Password with which to connect to the restore
+                                database. Defaults to the "password" option.
+                            example: trustsome1
                         authentication_database:
                             type: string
                             description: |
@@ -1216,7 +1303,7 @@ properties:
                         type: string
                         description: |
                             Healthchecks ping URL or UUID to notify when a
-                            backup begins, ends, errors or just to send logs.
+                            backup begins, ends, errors, or to send only logs.
                         example: https://hc-ping.com/your-uuid-here
                     verify_tls:
                         type: boolean

borgmatic/config/validate.py

@@ -1,9 +1,9 @@
 import os
 
 import jsonschema
-import pkg_resources
 import ruamel.yaml
 
+import borgmatic.config
 from borgmatic.config import environment, load, normalize, override
 
 
@@ -11,8 +11,13 @@ def schema_filename():
     '''
     Path to the installed YAML configuration schema file, used to validate and parse the
     configuration.
+
+    Raise FileNotFoundError when the schema path does not exist.
     '''
-    return pkg_resources.resource_filename('borgmatic', 'config/schema.yaml')
+    schema_path = os.path.join(os.path.dirname(borgmatic.config.__file__), 'schema.yaml')
+
+    with open(schema_path):
+        return schema_path
 
 
 def format_json_error_path_element(path_element):

borgmatic/execute.py

@@ -236,7 +236,11 @@ def execute_command(
 
 
 def execute_command_and_capture_output(
-    full_command, capture_stderr=False, shell=False, extra_environment=None, working_directory=None,
+    full_command,
+    capture_stderr=False,
+    shell=False,
+    extra_environment=None,
+    working_directory=None,
 ):
     '''
     Execute the given command (a sequence of command/argument strings), capturing and returning its

borgmatic/hooks/mongodb.py

@@ -102,7 +102,9 @@ def make_database_dump_pattern(
     return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
 
 
-def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
+def restore_database_dump(
+    database_config, log_prefix, location_config, dry_run, extract_process, connection_params
+):
     '''
     Restore the given MongoDB database from an extract stream. The database is supplied as a
     one-element sequence containing a dict describing the database, as per the configuration schema.
@@ -122,7 +124,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
     dump_filename = dump.make_database_dump_filename(
         make_dump_path(location_config), database['name'], database.get('hostname')
     )
-    restore_command = build_restore_command(extract_process, database, dump_filename)
+    restore_command = build_restore_command(
+        extract_process, database, dump_filename, connection_params
+    )
 
     logger.debug(f"{log_prefix}: Restoring MongoDB database {database['name']}{dry_run_label}")
     if dry_run:
@@ -138,10 +142,21 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
     )
 
 
-def build_restore_command(extract_process, database, dump_filename):
+def build_restore_command(extract_process, database, dump_filename, connection_params):
     '''
     Return the mongorestore command from a single database configuration.
     '''
+    hostname = connection_params['hostname'] or database.get(
+        'restore_hostname', database.get('hostname')
+    )
+    port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
+    username = connection_params['username'] or database.get(
+        'restore_username', database.get('username')
+    )
+    password = connection_params['password'] or database.get(
+        'restore_password', database.get('password')
+    )
+
     command = ['mongorestore']
     if extract_process:
         command.append('--archive')
@@ -149,16 +164,19 @@ def build_restore_command(extract_process, database, dump_filename):
         command.extend(('--dir', dump_filename))
     if database['name'] != 'all':
         command.extend(('--drop', '--db', database['name']))
-    if 'hostname' in database:
-        command.extend(('--host', database['hostname']))
-    if 'port' in database:
-        command.extend(('--port', str(database['port'])))
-    if 'username' in database:
-        command.extend(('--username', database['username']))
-    if 'password' in database:
-        command.extend(('--password', database['password']))
+    if hostname:
+        command.extend(('--host', hostname))
+    if port:
+        command.extend(('--port', str(port)))
+    if username:
+        command.extend(('--username', username))
+    if password:
+        command.extend(('--password', password))
     if 'authentication_database' in database:
         command.extend(('--authenticationDatabase', database['authentication_database']))
     if 'restore_options' in database:
         command.extend(database['restore_options'].split(' '))
+    if database['schemas']:
+        for schema in database['schemas']:
+            command.extend(('--nsInclude', schema))
     return command

borgmatic/hooks/mysql.py

@@ -100,7 +100,9 @@ def execute_dump_command(
     dump.create_named_pipe_for_dump(dump_filename)
 
     return execute_command(
-        dump_command, extra_environment=extra_environment, run_to_completion=False,
+        dump_command,
+        extra_environment=extra_environment,
+        run_to_completion=False,
     )
 
 
@@ -183,7 +185,9 @@ def make_database_dump_pattern(
     return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
 
 
-def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
+def restore_database_dump(
+    database_config, log_prefix, location_config, dry_run, extract_process, connection_params
+):
     '''
     Restore the given MySQL/MariaDB database from an extract stream. The database is supplied as a
     one-element sequence containing a dict describing the database, as per the configuration schema.
@@ -197,15 +201,27 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
         raise ValueError('The database configuration value is invalid')
 
     database = database_config[0]
+
+    hostname = connection_params['hostname'] or database.get(
+        'restore_hostname', database.get('hostname')
+    )
+    port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
+    username = connection_params['username'] or database.get(
+        'restore_username', database.get('username')
+    )
+    password = connection_params['password'] or database.get(
+        'restore_password', database.get('password')
+    )
+
     restore_command = (
         ('mysql', '--batch')
         + (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
-        + (('--host', database['hostname']) if 'hostname' in database else ())
-        + (('--port', str(database['port'])) if 'port' in database else ())
-        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
-        + (('--user', database['username']) if 'username' in database else ())
+        + (('--host', hostname) if hostname else ())
+        + (('--port', str(port)) if port else ())
+        + (('--protocol', 'tcp') if hostname or port else ())
+        + (('--user', username) if username else ())
     )
-    extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
+    extra_environment = {'MYSQL_PWD': password} if password else None
 
     logger.debug(f"{log_prefix}: Restoring MySQL database {database['name']}{dry_run_label}")
     if dry_run:

borgmatic/hooks/postgresql.py

@@ -1,6 +1,8 @@
 import csv
+import itertools
 import logging
 import os
+import shlex
 
 from borgmatic.execute import (
     execute_command,
@@ -21,13 +23,23 @@ def make_dump_path(location_config):  # pragma: no cover
     )
 
 
-def make_extra_environment(database):
+def make_extra_environment(database, restore_connection_params=None):
     '''
     Make the extra_environment dict from the given database configuration.
+    If restore connection params are given, this is for a restore operation.
     '''
     extra = dict()
-    if 'password' in database:
-        extra['PGPASSWORD'] = database['password']
+
+    try:
+        if restore_connection_params:
+            extra['PGPASSWORD'] = restore_connection_params.get('password') or database.get(
+                'restore_password', database['password']
+            )
+        else:
+            extra['PGPASSWORD'] = database['password']
+    except (AttributeError, KeyError):
+        pass
+
     extra['PGSSLMODE'] = database.get('ssl_mode', 'disable')
     if 'ssl_cert' in database:
         extra['PGSSLCERT'] = database['ssl_cert']
@@ -59,8 +71,10 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run):
     if dry_run:
         return ()
 
+    psql_command = shlex.split(database.get('psql_command') or 'psql')
     list_command = (
-        ('psql', '--list', '--no-password', '--csv', '--tuples-only')
+        tuple(psql_command)
+        + ('--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only')
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--username', database['username']) if 'username' in database else ())
@@ -122,10 +136,16 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
                 continue
 
             command = (
-                (dump_command, '--no-password', '--clean', '--if-exists',)
+                (
+                    dump_command,
+                    '--no-password',
+                    '--clean',
+                    '--if-exists',
+                )
                 + (('--host', database['hostname']) if 'hostname' in database else ())
                 + (('--port', str(database['port'])) if 'port' in database else ())
                 + (('--username', database['username']) if 'username' in database else ())
+                + (('--no-owner',) if database.get('no_owner', False) else ())
                 + (('--format', dump_format) if dump_format else ())
                 + (('--file', dump_filename) if dump_format == 'directory' else ())
                 + (tuple(database['options'].split(' ')) if 'options' in database else ())
@@ -145,7 +165,9 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
             if dump_format == 'directory':
                 dump.create_parent_directory_for_dump(dump_filename)
                 execute_command(
-                    command, shell=True, extra_environment=extra_environment,
+                    command,
+                    shell=True,
+                    extra_environment=extra_environment,
                 )
             else:
                 dump.create_named_pipe_for_dump(dump_filename)
@@ -181,7 +203,9 @@ def make_database_dump_pattern(
     return dump.make_database_dump_filename(make_dump_path(location_config), name, hostname='*')
 
 
-def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
+def restore_database_dump(
+    database_config, log_prefix, location_config, dry_run, extract_process, connection_params
+):
     '''
     Restore the given PostgreSQL database from an extract stream. The database is supplied as a
     one-element sequence containing a dict describing the database, as per the configuration schema.
@@ -191,6 +215,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
 
     If the extract process is None, then restore the dump from the filesystem rather than from an
     extract stream.
+
+    Use the given connection parameters to connect to the database. The connection parameters are
+    hostname, port, username, and password.
     '''
     dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
 
@@ -198,35 +225,53 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
         raise ValueError('The database configuration value is invalid')
 
     database = database_config[0]
+
+    hostname = connection_params['hostname'] or database.get(
+        'restore_hostname', database.get('hostname')
+    )
+    port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
+    username = connection_params['username'] or database.get(
+        'restore_username', database.get('username')
+    )
+
     all_databases = bool(database['name'] == 'all')
     dump_filename = dump.make_database_dump_filename(
         make_dump_path(location_config), database['name'], database.get('hostname')
     )
-    psql_command = database.get('psql_command') or 'psql'
+    psql_command = shlex.split(database.get('psql_command') or 'psql')
     analyze_command = (
-        (psql_command, '--no-password', '--quiet')
-        + (('--host', database['hostname']) if 'hostname' in database else ())
-        + (('--port', str(database['port'])) if 'port' in database else ())
-        + (('--username', database['username']) if 'username' in database else ())
+        tuple(psql_command)
+        + ('--no-password', '--no-psqlrc', '--quiet')
+        + (('--host', hostname) if hostname else ())
+        + (('--port', port) if port else ())
+        + (('--username', username) if username else ())
         + (('--dbname', database['name']) if not all_databases else ())
         + (tuple(database['analyze_options'].split(' ')) if 'analyze_options' in database else ())
         + ('--command', 'ANALYZE')
     )
-    pg_restore_command = database.get('pg_restore_command') or 'pg_restore'
+    use_psql_command = all_databases or database.get('format') == 'plain'
+    pg_restore_command = shlex.split(database.get('pg_restore_command') or 'pg_restore')
     restore_command = (
-        (psql_command if all_databases else pg_restore_command, '--no-password')
-        + (
-            ('--if-exists', '--exit-on-error', '--clean', '--dbname', database['name'])
-            if not all_databases
-            else ()
-        )
-        + (('--host', database['hostname']) if 'hostname' in database else ())
-        + (('--port', str(database['port'])) if 'port' in database else ())
-        + (('--username', database['username']) if 'username' in database else ())
+        tuple(psql_command if use_psql_command else pg_restore_command)
+        + ('--no-password',)
+        + (('--no-psqlrc',) if use_psql_command else ('--if-exists', '--exit-on-error', '--clean'))
+        + (('--dbname', database['name']) if not all_databases else ())
+        + (('--host', hostname) if hostname else ())
+        + (('--port', port) if port else ())
+        + (('--username', username) if username else ())
+        + (('--no-owner',) if database.get('no_owner', False) else ())
         + (tuple(database['restore_options'].split(' ')) if 'restore_options' in database else ())
         + (() if extract_process else (dump_filename,))
+        + tuple(
+            itertools.chain.from_iterable(('--schema', schema) for schema in database['schemas'])
+            if database['schemas']
+            else ()
+        )
+    )
+
+    extra_environment = make_extra_environment(
+        database, restore_connection_params=connection_params
     )
-    extra_environment = make_extra_environment(database)
 
     logger.debug(f"{log_prefix}: Restoring PostgreSQL database {database['name']}{dry_run_label}")
     if dry_run:

borgmatic/hooks/sqlite.py

@@ -85,7 +85,9 @@ def make_database_dump_pattern(
     return dump.make_database_dump_filename(make_dump_path(location_config), name)
 
 
-def restore_database_dump(database_config, log_prefix, location_config, dry_run, extract_process):
+def restore_database_dump(
+    database_config, log_prefix, location_config, dry_run, extract_process, connection_params
+):
     '''
     Restore the given SQLite3 database from an extract stream. The database is supplied as a
     one-element sequence containing a dict describing the database, as per the configuration schema.
@@ -98,7 +100,9 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
     if len(database_config) != 1:
         raise ValueError('The database configuration value is invalid')
 
-    database_path = database_config[0]['path']
+    database_path = connection_params['restore_path'] or database_config[0].get(
+        'restore_path', database_config[0].get('path')
+    )
 
     logger.debug(f'{log_prefix}: Restoring SQLite database at {database_path}{dry_run_label}')
     if dry_run:

borgmatic/logger.py

@@ -141,6 +141,7 @@ def add_logging_level(level_name, level_number):
 
 
 ANSWER = logging.WARN - 5
+DISABLED = logging.CRITICAL + 10
 
 
 def add_custom_log_levels():  # pragma: no cover
@@ -148,6 +149,7 @@ def add_custom_log_levels():  # pragma: no cover
     Add a custom log level between WARN and INFO for user-requested answers.
     '''
     add_logging_level('ANSWER', ANSWER)
+    add_logging_level('DISABLED', DISABLED)
 
 
 def configure_logging(
@@ -175,10 +177,12 @@ def configure_logging(
 
     # Log certain log levels to console stderr and others to stdout. This supports use cases like
     # grepping (non-error) output.
+    console_disabled = logging.NullHandler()
     console_error_handler = logging.StreamHandler(sys.stderr)
     console_standard_handler = logging.StreamHandler(sys.stdout)
     console_handler = Multi_stream_handler(
         {
+            logging.DISABLED: console_disabled,
             logging.CRITICAL: console_error_handler,
             logging.ERROR: console_error_handler,
             logging.WARN: console_error_handler,
@@ -191,7 +195,7 @@ def configure_logging(
     console_handler.setLevel(console_log_level)
 
     syslog_path = None
-    if log_file is None:
+    if log_file is None and syslog_log_level != logging.DISABLED:
         if os.path.exists('/dev/log'):
             syslog_path = '/dev/log'
         elif os.path.exists('/var/run/syslog'):
@@ -206,7 +210,7 @@ def configure_logging(
         )
         syslog_handler.setLevel(syslog_log_level)
         handlers = (console_handler, syslog_handler)
-    elif log_file:
+    elif log_file and log_file_log_level != logging.DISABLED:
         file_handler = logging.handlers.WatchedFileHandler(log_file)
         file_handler.setFormatter(
             logging.Formatter(

borgmatic/verbosity.py

@@ -2,6 +2,7 @@ import logging
 
 import borgmatic.logger
 
+VERBOSITY_DISABLED = -2
 VERBOSITY_ERROR = -1
 VERBOSITY_ANSWER = 0
 VERBOSITY_SOME = 1
@@ -15,6 +16,7 @@ def verbosity_to_log_level(verbosity):
     borgmatic.logger.add_custom_log_levels()
 
     return {
+        VERBOSITY_DISABLED: logging.DISABLED,
         VERBOSITY_ERROR: logging.ERROR,
         VERBOSITY_ANSWER: logging.ANSWER,
         VERBOSITY_SOME: logging.INFO,

docs/Dockerfile

@@ -4,9 +4,9 @@ COPY . /app
 RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib
 RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
 RUN borgmatic --help > /command-line.txt \
-    && for action in rcreate transfer create prune compact check extract export-tar mount umount restore rlist list rinfo info break-lock borg; do \
+    && for action in rcreate transfer create prune compact check extract config "config bootstrap" "config generate" "config validate" export-tar mount umount restore rlist list rinfo info break-lock borg; do \
            echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \
-           && borgmatic "$action" --help >> /command-line.txt; done
+           && borgmatic $action --help >> /command-line.txt; done
 
 FROM docker.io/node:19.5.0-alpine as html
 

docs/docker-compose.yaml

@@ -0,0 +1,22 @@
+version: '3'
+services:
+  docs:
+    image: borgmatic-docs
+    container_name: docs
+    ports:
+      - 8080:80
+    build:
+      dockerfile: docs/Dockerfile
+      context: ..
+      args:
+        ENVIRONMENT: dev
+  message:
+    image: alpine
+    container_name: message
+    command:
+      - sh
+      - -c
+      - |
+        echo; echo "You can view dev docs at http://localhost:8080"; echo
+    depends_on:
+      - docs

docs/how-to/add-preparation-and-cleanup-steps-to-backups.md

@@ -28,6 +28,16 @@ hooks:
         - umount /some/filesystem
 ```
 
+If your command contains a special YAML character such as a colon, you may
+need to quote the entire string (or use a [multiline
+string](https://yaml-multiline.info/)) to avoid an error:
+
+```yaml
+hooks:
+    before_backup:
+        - "echo Backup: start"
+```
+
 <span class="minilink minilink-addedin">New in version 1.6.0</span> The
 `before_backup` and `after_backup` hooks each run once per repository in a
 configuration file. `before_backup` hooks runs right before the `create`

docs/how-to/backup-your-databases.md

@@ -138,9 +138,9 @@ hooks:
 
 ### Containers
 
-If your database is running within a Docker container and borgmatic is too, no
-problem—simply configure borgmatic to connect to the container's name on its
-exposed port. For instance:
+If your database is running within a container and borgmatic is too, no
+problem—configure borgmatic to connect to the container's name on its exposed
+port. For instance:
 
 ```yaml
 hooks:
@@ -154,10 +154,10 @@ hooks:
 
 But what if borgmatic is running on the host? You can still connect to a
 database container if its ports are properly exposed to the host. For
-instance, when running the database container with Docker, you can specify
-`--publish 127.0.0.1:5433:5432` so that it exposes the container's port 5432
-to port 5433 on the host (only reachable on localhost, in this case). Or the
-same thing with Docker Compose:
+instance, when running the database container, you can specify `--publish
+127.0.0.1:5433:5432` so that it exposes the container's port 5432 to port 5433
+on the host (only reachable on localhost, in this case). Or the same thing
+with Docker Compose:
 
 ```yaml
 services:
@@ -179,7 +179,7 @@ hooks:
           password: trustsome1
 ```
 
-Of course, alter the ports in these examples to suit your particular database
+You can alter the ports in these examples to suit your particular database
 system.
 
 
@@ -324,6 +324,17 @@ includes any combined dump file named "all" and any other individual database
 dumps found in the archive.
 
 
+### Restore particular schemas
+
+<span class="minilink minilink-addedin">New in version 1.7.13</span> With
+PostgreSQL and MongoDB, you can limit the restore to a single schema found
+within the database dump:
+
+```bash
+borgmatic restore --archive latest --database users --schema tentant1
+```
+
+
 ### Limitations
 
 There are a few important limitations with borgmatic's current database
@@ -337,17 +348,19 @@ borgmatic's own configuration file. So include your configuration file in
 backups to avoid getting caught without a way to restore a database.
 3. borgmatic does not currently support backing up or restoring multiple
 databases that share the exact same name on different hosts.
-4. Because database hooks implicitly enable the `read_special` configuration
-setting to support dump and restore streaming, you'll need to ensure that any
-special files are excluded from backups (named pipes, block devices,
-character devices, and sockets) to prevent hanging. Try a command like
-`find /your/source/path -type b -or -type c -or -type p -or -type s` to find
-such files. Common directories to exclude are `/dev` and `/run`, but that may
-not be exhaustive. <span class="minilink minilink-addedin">New in version
+4. Because database hooks implicitly enable the `read_special` configuration,
+any special files are excluded from backups (named pipes, block devices,
+character devices, and sockets) to prevent hanging. Try a command like `find
+/your/source/path -type b -or -type c -or -type p -or -type s` to find such
+files. Common directories to exclude are `/dev` and `/run`, but that may not
+be exhaustive. <span class="minilink minilink-addedin">New in version
 1.7.3</span> When database hooks are enabled, borgmatic automatically excludes
-special files that may cause Borg to hang, so you no longer need to manually
-exclude them. (This includes symlinks with special files as a destination.) You
-can override/prevent this behavior by explicitly setting `read_special` to true.
+special files (and symlinks to special files) that may cause Borg to hang, so
+generally you no longer need to manually exclude them. There are potential
+edge cases though in which applications on your system create new special files
+*after* borgmatic constructs its exclude list, resulting in Borg hangs. If that
+occurs, you can resort to the manual excludes described above. And to opt out
+of the auto-exclude feature entirely, explicitly set `read_special` to true.
 
 
 ### Manual restoration
@@ -386,9 +399,9 @@ dumps with any database system.
 With PostgreSQL and MySQL/MariaDB, if you're getting authentication errors
 when borgmatic tries to connect to your database, a natural reaction is to
 increase your borgmatic verbosity with `--verbosity 2` and go looking in the
-logs. You'll notice however that your database password does not show up in
-the logs. This is likely not the cause of the authentication problem unless
-you mistyped your password, however; borgmatic passes your password to the
+logs. You'll notice though that your database password does not show up in the
+logs. This is likely not the cause of the authentication problem unless you
+mistyped your password, however; borgmatic passes your password to the
 database via an environment variable that does not appear in the logs.
 
 The cause of an authentication error is often on the database side—in the
@@ -397,6 +410,12 @@ authenticated. For instance, with PostgreSQL, check your
 [pg_hba.conf](https://www.postgresql.org/docs/current/auth-pg-hba-conf.html)
 file for that configuration.
 
+Additionally, MySQL/MariaDB may be picking up some of your credentials from a
+defaults file like `~/.my.cnf`. If that's the case, then it's possible
+MySQL/MariaDB ends up using, say, a username from borgmatic's configuration
+and a password from `~/.my.cnf`. This may result in authentication errors if
+this combination of credentials is not what you intend.
+
 
 ### MySQL table lock errors
 

docs/how-to/deal-with-very-large-backups.md

@@ -95,6 +95,7 @@ See [Borg's check
 documentation](https://borgbackup.readthedocs.io/en/stable/usage/check.html)
 for more information.
 
+
 ### Check frequency
 
 <span class="minilink minilink-addedin">New in version 1.6.2</span> You can

docs/how-to/develop-on-borgmatic.md

@@ -7,7 +7,7 @@ eleventyNavigation:
 ---
 ## Source code
 
-To get set up to hack on borgmatic, first clone master via HTTPS or SSH:
+To get set up to hack on borgmatic, first clone it via HTTPS or SSH:
 
 ```bash
 git clone https://projects.torsion.org/borgmatic-collective/borgmatic.git
@@ -87,19 +87,20 @@ tox -e codespell
 borgmatic additionally includes some end-to-end tests that integration test
 with Borg and supported databases for a few representative scenarios. These
 tests don't run by default when running `tox`, because they're relatively slow
-and depend on Docker containers for runtime dependencies. These tests tests do
-run on the continuous integration (CI) server, and running them on your
-developer machine is the closest thing to CI test parity.
+and depend on containers for runtime dependencies. These tests do run on the
+continuous integration (CI) server, and running them on your developer machine
+is the closest thing to CI-test parity.
 
-If you would like to run the full test suite, first install Docker and [Docker
-Compose](https://docs.docker.com/compose/install/). Then run:
+If you would like to run the full test suite, first install Docker (or Podman;
+see below) and [Docker Compose](https://docs.docker.com/compose/install/).
+Then run:
 
 ```bash
 scripts/run-end-to-end-dev-tests
 ```
 
-Note that this scripts assumes you have permission to run Docker. If you
-don't, then you may need to run with `sudo`.
+This script assumes you have permission to run `docker`. If you don't, then
+you may need to run with `sudo`.
 
 
 #### Podman
@@ -112,13 +113,13 @@ borgmatic's end-to-end tests optionally support using
 Setting up Podman is outside the scope of this documentation, but here are
 some key points to double-check:
 
- * Install Podman along with `podman-docker` and your desired networking
-   support.
+ * Install Podman and your desired networking support.
  * Configure `/etc/subuid` and `/etc/subgid` to map users/groups for the
    non-root user who will run tests.
  * Create a non-root Podman socket for that user:
    ```bash
    systemctl --user enable --now podman.socket
+   systemctl --user start --now podman.socket
    ```
 
 Then you'll be able to run end-to-end tests as per normal, and the test script
@@ -161,11 +162,12 @@ To build and view a copy of the documentation with your local changes, run the
 following from the root of borgmatic's source code:
 
 ```bash
-sudo scripts/dev-docs
+scripts/dev-docs
 ```
 
-This requires Docker to be installed on your system. You may not need to use
-sudo if your non-root user has permissions to run Docker.
+This requires Docker (or Podman; see below) to be installed on your system.
+This script assumes you have permission to run `docker`. If you don't, then
+you may need to run with `sudo`.
 
 After you run the script, you can point your web browser at
 http://localhost:8080 to view the documentation with your changes.
@@ -183,5 +185,5 @@ borgmatic's developer build for documentation optionally supports using
 [Podman](https://podman.io/) instead of Docker.
 
 Setting up Podman is outside the scope of this documentation. But once you
-install `podman-docker`, then `scripts/dev-docs` should automatically use
+install and configure Podman, then `scripts/dev-docs` should automatically use
 Podman instead of Docker.

docs/how-to/inspect-your-backups.md

@@ -24,6 +24,15 @@ Or, for even more progress and debug spew:
 borgmatic --verbosity 2
 ```
 
+The full set of verbosity levels are:
+
+ * `-2`: disable output entirely <span class="minilink minilink-addedin">New in borgmatic 1.7.14</span>
+ * `-1`: only show errors
+ * `0`: default output
+ * `1`: some additional output (informational level)
+ * `2`: lots of additional output (debug level)
+
+
 ## Backup summary
 
 If you're less concerned with progress during a backup, and you only want to
@@ -169,7 +178,7 @@ brackets. For instance, the default log format is: `[{asctime}] {levelname}:
 {message}`. This means each log message is recorded as the log time (in square
 brackets), a logging level name, a colon, and the actual log message.
 
-So if you just want each log message to get logged *without* a timestamp or a
+So if you only want each log message to get logged *without* a timestamp or a
 logging level name:
 
 ```bash

docs/how-to/make-per-application-backups.md

@@ -20,18 +20,22 @@ instance, for applications:
 
 ```bash
 sudo mkdir /etc/borgmatic.d
-sudo generate-borgmatic-config --destination /etc/borgmatic.d/app1.yaml
-sudo generate-borgmatic-config --destination /etc/borgmatic.d/app2.yaml
+sudo borgmatic config generate --destination /etc/borgmatic.d/app1.yaml
+sudo borgmatic config generate --destination /etc/borgmatic.d/app2.yaml
 ```
 
 Or, for repositories:
 
 ```bash
 sudo mkdir /etc/borgmatic.d
-sudo generate-borgmatic-config --destination /etc/borgmatic.d/repo1.yaml
-sudo generate-borgmatic-config --destination /etc/borgmatic.d/repo2.yaml
+sudo borgmatic config generate --destination /etc/borgmatic.d/repo1.yaml
+sudo borgmatic config generate --destination /etc/borgmatic.d/repo2.yaml
 ```
 
+<span class="minilink minilink-addedin">Prior to version 1.7.15</span> The
+command to generate configuration files was `generate-borgmatic-config`
+instead of `borgmatic config generate`.
+
 When you set up multiple configuration files like this, borgmatic will run
 each one in turn from a single borgmatic invocation. This includes, by
 default, the traditional `/etc/borgmatic/config.yaml` as well.
@@ -81,13 +85,16 @@ If `archive_name_format` is unspecified, the default is
 `{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}`, meaning your system hostname plus a
 timestamp in a particular format.
 
+
+### Archive filtering
+
 <span class="minilink minilink-addedin">New in version 1.7.11</span> borgmatic
 uses the `archive_name_format` option to automatically limit which archives
 get used for actions operating on multiple archives. This prevents, for
 instance, duplicate archives from showing up in `rlist` or `info` results—even
 if the same repository appears in multiple borgmatic configuration files. To
-take advantage of this feature, simply use a different `archive_name_format`
-in each configuration file.
+take advantage of this feature, use a different `archive_name_format` in each
+configuration file.
 
 Under the hood, borgmatic accomplishes this by substituting globs for certain
 ephemeral data placeholders in your `archive_name_format`—and using the result
@@ -108,8 +115,8 @@ archives used for actions like `rlist`, `info`, `prune`, `check`, etc.
 
 The end result is that when borgmatic runs the actions for a particular
 application-specific configuration file, it only operates on the archives
-created for that application. Of course, this doesn't apply to actions like
-`compact` that operate on an entire repository.
+created for that application. But this doesn't apply to actions like `compact`
+that operate on an entire repository.
 
 If this behavior isn't quite smart enough for your needs, you can use the
 `match_archives` option to override the pattern that borgmatic uses for
@@ -392,9 +399,16 @@ includes.
 
 ## Debugging includes
 
-<span class="minilink minilink-addedin">New in version 1.7.12</span> If you'd
+<span class="minilink minilink-addedin">New in version 1.7.15</span> If you'd
 like to see what the loaded configuration looks like after includes get merged
-in, run `validate-borgmatic-config` on your configuration file:
+in, run the `validate` action on your configuration file:
+
+```bash
+sudo borgmatic config validate --show
+```
+
+<span class="minilink minilink-addedin">In version 1.7.12 through
+1.7.14</span> Use this command instead:
 
 ```bash
 sudo validate-borgmatic-config --show

docs/how-to/monitor-your-backups.md

@@ -73,7 +73,7 @@ from borgmatic for a configured interval.
 
 ### Consistency checks
 
-While not strictly part of monitoring, if you really want confidence that your
+While not strictly part of monitoring, if you want confidence that your
 backups are not only running but are restorable as well, you can configure
 particular [consistency
 checks](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/#consistency-check-configuration)

docs/how-to/run-arbitrary-borg-commands.md

@@ -89,8 +89,10 @@ borgmatic's `borg` action is not without limitations:
  * Unlike normal borgmatic actions that support JSON, the `borg` action will
    not disable certain borgmatic logs to avoid interfering with JSON output.
  * Unlike other borgmatic actions, the `borg` action captures (and logs) all
-   output, so interactive prompts or flags like `--progress` will not work as
-   expected.
+   output, so interactive prompts and flags like `--progress` will not work as
+   expected. <span class="minilink minilink-addedin">New in version
+   1.7.13</span> borgmatic now runs the `borg` action without capturing output,
+   so interactive prompts work.
 
 In general, this `borgmatic borg` feature should be considered an escape
 valve—a feature of second resort. In the long run, it's preferable to wrap

docs/how-to/set-up-backups.md

@@ -82,13 +82,14 @@ on a relatively dedicated system, then a global install can work out fine.
 Besides the approaches described above, there are several other options for
 installing borgmatic:
 
- * [Docker image with scheduled backups](https://hub.docker.com/r/b3vis/borgmatic/) (+ Docker Compose files)
- * [Docker image with multi-arch and Docker CLI support](https://hub.docker.com/r/modem7/borgmatic-docker/)
+ * [container image with scheduled backups](https://hub.docker.com/r/b3vis/borgmatic/) (+ Docker Compose files)
+ * [container image with multi-arch and Docker CLI support](https://hub.docker.com/r/modem7/borgmatic-docker/)
  * [Debian](https://tracker.debian.org/pkg/borgmatic)
  * [Ubuntu](https://launchpad.net/ubuntu/+source/borgmatic)
  * [Fedora official](https://bodhi.fedoraproject.org/updates/?search=borgmatic)
  * [Fedora unofficial](https://copr.fedorainfracloud.org/coprs/heffer/borgmatic/)
- * [Arch Linux](https://www.archlinux.org/packages/community/any/borgmatic/)
+ * [Gentoo](https://packages.gentoo.org/packages/app-backup/borgmatic)
+ * [Arch Linux](https://archlinux.org/packages/extra/any/borgmatic/)
  * [Alpine Linux](https://pkgs.alpinelinux.org/packages?name=borgmatic)
  * [OpenBSD](https://openports.pl/path/sysutils/borgmatic)
  * [openSUSE](https://software.opensuse.org/package/borgmatic)
@@ -96,7 +97,7 @@ installing borgmatic:
  * [macOS (via MacPorts)](https://ports.macports.org/port/borgmatic/)
  * [NixOS](https://search.nixos.org/packages?show=borgmatic&sort=relevance&type=packages&query=borgmatic)
  * [Ansible role](https://github.com/borgbase/ansible-role-borgbackup)
- * [virtualenv](https://virtualenv.pypa.io/en/stable/)
+ * [Unraid](https://unraid.net/community/apps?q=borgmatic#r)
 
 
 ## Hosting providers
@@ -119,16 +120,24 @@ offerings, but do not currently fund borgmatic development or hosting.
 
 After you install borgmatic, generate a sample configuration file:
 
+```bash
+sudo borgmatic config generate
+```
+
+<span class="minilink minilink-addedin">Prior to version 1.7.15</span>
+Generate a configuration file with this command instead:
+
 ```bash
 sudo generate-borgmatic-config
 ```
 
-If that command is not found, then it may be installed in a location that's
-not in your system `PATH` (see above). Try looking in `~/.local/bin/`.
+If neither command is found, then borgmatic may be installed in a location
+that's not in your system `PATH` (see above). Try looking in `~/.local/bin/`.
 
-This generates a sample configuration file at `/etc/borgmatic/config.yaml` by
-default. If you'd like to use another path, use the `--destination` flag, for
-instance: `--destination ~/.config/borgmatic/config.yaml`.
+The command generates a sample configuration file at
+`/etc/borgmatic/config.yaml` by default. If you'd like to use another path,
+use the `--destination` flag, for instance: `--destination
+~/.config/borgmatic/config.yaml`.
 
 You should edit the configuration file to suit your needs, as the generated
 values are only representative. All options are optional except where
@@ -176,6 +185,13 @@ redundant](https://torsion.org/borgmatic/docs/how-to/make-backups-redundant/).
 If you'd like to validate that your borgmatic configuration is valid, the
 following command is available for that:
 
+```bash
+sudo borgmatic config validate
+```
+
+<span class="minilink minilink-addedin">Prior to version 1.7.15</span>
+Validate a configuration file with this command instead:
+
 ```bash
 sudo validate-borgmatic-config
 ```
@@ -279,7 +295,7 @@ that, you can configure a separate job runner to invoke it periodically.
 ### cron
 
 If you're using cron, download the [sample cron
-file](https://projects.torsion.org/borgmatic-collective/borgmatic/src/master/sample/cron/borgmatic).
+file](https://projects.torsion.org/borgmatic-collective/borgmatic/src/main/sample/cron/borgmatic).
 Then, from the directory where you downloaded it:
 
 ```bash
@@ -303,9 +319,9 @@ you may already have borgmatic systemd service and timer files. If so, you may
 be able to skip some of the steps below.)
 
 First, download the [sample systemd service
-file](https://projects.torsion.org/borgmatic-collective/borgmatic/raw/branch/master/sample/systemd/borgmatic.service)
+file](https://projects.torsion.org/borgmatic-collective/borgmatic/raw/branch/main/sample/systemd/borgmatic.service)
 and the [sample systemd timer
-file](https://projects.torsion.org/borgmatic-collective/borgmatic/raw/branch/master/sample/systemd/borgmatic.timer).
+file](https://projects.torsion.org/borgmatic-collective/borgmatic/raw/branch/main/sample/systemd/borgmatic.timer).
 
 Then, from the directory where you downloaded them:
 
@@ -334,10 +350,13 @@ Access](https://projects.torsion.org/borgmatic-collective/borgmatic/issues/293).
 
 ### Shell completion
 
-borgmatic includes a shell completion script (currently only for Bash) to
+borgmatic includes a shell completion script (currently only for Bash and Fish) to
 support tab-completing borgmatic command-line actions and flags. Depending on
-how you installed borgmatic, this may be enabled by default. But if it's not,
-start by installing the `bash-completion` Linux package or the
+how you installed borgmatic, this may be enabled by default.
+
+#### Bash
+
+If completions aren't enabled, start by installing the `bash-completion` Linux package or the
 [`bash-completion@2`](https://formulae.brew.sh/formula/bash-completion@2)
 macOS Homebrew formula. Then, install the shell completion script globally:
 
@@ -362,6 +381,14 @@ borgmatic --bash-completion > ~/.local/share/bash-completion/completions/borgmat
 Finally, restart your shell (`exit` and open a new shell) so the completions
 take effect.
 
+#### fish
+
+To add completions for fish, install the completions file globally:
+
+```fish
+borgmatic --fish-completion | sudo tee /usr/share/fish/vendor_completions.d/borgmatic.fish
+source /usr/share/fish/vendor_completions.d/borgmatic.fish
+```
 
 ### Colored output
 

docs/how-to/upgrade.md

@@ -29,29 +29,33 @@ configuration options. This is completely optional. If you prefer, you can add
 new configuration options manually.
 
 If you do want to upgrade your configuration file to include new options, use
-the `generate-borgmatic-config` script with its optional `--source` flag that
+the `borgmatic config generate` action with its optional `--source` flag that
 takes the path to your original configuration file. If provided with this
-path, `generate-borgmatic-config` merges your original configuration into the
+path, `borgmatic config generate` merges your original configuration into the
 generated configuration file, so you get all the newest options and comments.
 
 Here's an example:
 
 ```bash
-generate-borgmatic-config --source config.yaml --destination config-new.yaml
+borgmatic config generate --source config.yaml --destination config-new.yaml
 ```
 
+<span class="minilink minilink-addedin">Prior to version 1.7.15</span> The
+command to generate configuration files was `generate-borgmatic-config`
+instead of `borgmatic config generate`.
+
 New options start as commented out, so you can edit the file and decide
 whether you want to use each one.
 
 There are a few caveats to this process. First, when generating the new
-configuration file, `generate-borgmatic-config` replaces any comments you've
+configuration file, `borgmatic config generate` replaces any comments you've
 written in your original configuration file with the newest generated
 comments. Second, the script adds back any options you had originally deleted,
 although it does so with the options commented out. And finally, any YAML
 includes you've used in the source configuration get flattened out into a
 single generated file.
 
-As a safety measure, `generate-borgmatic-config` refuses to modify
+As a safety measure, `borgmatic config generate` refuses to modify
 configuration files in-place. So it's up to you to review the generated file
 and, if desired, replace your original configuration file with it.
 
@@ -61,21 +65,22 @@ and, if desired, replace your original configuration file with it.
 borgmatic changed its configuration file format in version 1.1.0 from
 INI-style to YAML. This better supports validation, and has a more natural way
 to express lists of values. To upgrade your existing configuration, first
-upgrade to the new version of borgmatic.
+upgrade to the last version of borgmatic to support converting configuration:
+borgmatic 1.7.14.
 
 As of version 1.1.0, borgmatic no longer supports Python 2. If you were
 already running borgmatic with Python 3, then you can upgrade borgmatic
 in-place:
 
 ```bash
-sudo pip3 install --user --upgrade borgmatic
+sudo pip3 install --user --upgrade borgmatic==1.7.14
 ```
 
 But if you were running borgmatic with Python 2, uninstall and reinstall instead:
 
 ```bash
 sudo pip uninstall borgmatic
-sudo pip3 install --user borgmatic
+sudo pip3 install --user borgmatic==1.7.14
 ```
 
 The pip binary names for different versions of Python can differ, so the above
@@ -93,29 +98,12 @@ That will generate a new YAML configuration file at /etc/borgmatic/config.yaml
 excludes files. The new version of borgmatic will consume the YAML
 configuration file instead of the old one.
 
-
-### Upgrading from atticmatic
-
-You can ignore this section if you're not an atticmatic user (the former name
-of borgmatic).
-
-borgmatic only supports Borg now and no longer supports Attic. So if you're
-an Attic user, consider switching to Borg. See the [Borg upgrade
-command](https://borgbackup.readthedocs.io/en/stable/usage.html#borg-upgrade)
-for more information. Then, follow the instructions above about setting up
-your borgmatic configuration files.
-
-If you were already using Borg with atticmatic, then you can upgrade
-from atticmatic to borgmatic by running the following commands:
+Now you can upgrade to a newer version of borgmatic:
 
 ```bash
-sudo pip3 uninstall atticmatic
 sudo pip3 install --user borgmatic
 ```
 
-That's it! borgmatic will continue using your /etc/borgmatic configuration
-files.
-
 
 ## Upgrading Borg
 

docs/reference/command-line.md

@@ -7,11 +7,18 @@ eleventyNavigation:
 ---
 ## borgmatic options
 
-Here are all of the available borgmatic command-line options, including the
-separate options for each action sub-command. Note that most of the
-flags listed here do not have equivalents in borgmatic's [configuration
+Here are all of the available borgmatic command-line flags for the [most
+recent version of
+borgmatic](https://projects.torsion.org/borgmatic-collective/borgmatic/releases),
+including the separate flags for each action (sub-command). Most of the flags
+listed here do not have equivalents in borgmatic's [configuration
 file](https://torsion.org/borgmatic/docs/reference/configuration/).
 
+If you're using an older version of borgmatic, some of these flags may not be
+present in that version, and you should instead use `borgmatic --help` or
+`borgmatic [action name] --help` (where `[action name]` is the name of an
+action like `list`, `create`, etc.).
+
 ```
 {% include borgmatic/command-line.txt %}
 ```

docs/reference/configuration.md

@@ -7,11 +7,19 @@ eleventyNavigation:
 ---
 ## Configuration file
 
-Here is a full sample borgmatic configuration file including all available options: 
+Below is a sample borgmatic configuration file including all available options
+for the [most recent version of
+borgmatic](https://projects.torsion.org/borgmatic-collective/borgmatic/releases).
+This file is also [available for
+download](https://torsion.org/borgmatic/docs/reference/config.yaml).
+
+If you're using an older version of borgmatic, some of these options may not
+work, and you should instead [generate a sample configuration file specific to
+your borgmatic
+version](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#configuration).
 
 ```yaml
 {% include borgmatic/config.yaml %}
 ```
 
 Note that you can also [download this configuration
-file](https://torsion.org/borgmatic/docs/reference/config.yaml) for use locally.

sample/systemd/borgmatic-user.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=borgmatic backup
+Wants=network-online.target
+After=network-online.target
+ConditionACPower=true
+
+[Service]
+Type=oneshot
+Restart=no
+# Prevent rate limiting of borgmatic log events. If you are using an older version of systemd that
+# doesn't support this (pre-240 or so), you may have to remove this option.
+LogRateLimitIntervalSec=0
+# Delay start to prevent backups running during boot.
+ExecStartPre=sleep 1m
+ExecStart=/root/.local/bin/borgmatic --verbosity -2 --syslog-verbosity 1

sample/systemd/borgmatic-user.timer

@@ -0,0 +1 @@
+borgmatic.timer

sample/systemd/borgmatic.service

@@ -61,4 +61,4 @@ LogRateLimitIntervalSec=0
 # Delay start to prevent backups running during boot. Note that systemd-inhibit requires dbus and
 # dbus-user-session to be installed.
 ExecStartPre=sleep 1m
-ExecStart=systemd-inhibit --who="borgmatic" --what="sleep:shutdown" --why="Prevent interrupting scheduled backup" /root/.local/bin/borgmatic --verbosity -1 --syslog-verbosity 1
+ExecStart=systemd-inhibit --who="borgmatic" --what="sleep:shutdown" --why="Prevent interrupting scheduled backup" /root/.local/bin/borgmatic --verbosity -2 --syslog-verbosity 1

scripts/dev-docs

@@ -2,8 +2,10 @@
 
 set -e
 
-docker build --tag borgmatic-docs --build-arg ENVIRONMENT=dev --file docs/Dockerfile .
-echo
-echo "You can view dev docs at http://localhost:8080"
-echo
-docker run --interactive --tty --publish 8080:80 --rm borgmatic-docs
+USER_PODMAN_SOCKET_PATH=/run/user/$UID/podman/podman.sock
+
+if [ -e "$USER_PODMAN_SOCKET_PATH" ]; then
+    export DOCKER_HOST="unix://$USER_PODMAN_SOCKET_PATH"
+fi
+
+docker-compose --file docs/docker-compose.yaml up --build --force-recreate

scripts/release

@@ -35,10 +35,8 @@ git push github $version
 rm -fr dist
 python3 setup.py bdist_wheel
 python3 setup.py sdist
-gpg --detach-sign --armor dist/borgmatic-*.tar.gz
-gpg --detach-sign --armor dist/borgmatic-*-py3-none-any.whl
-twine upload -r pypi --username __token__ dist/borgmatic-*.tar.gz dist/borgmatic-*.tar.gz.asc
-twine upload -r pypi --username __token__ dist/borgmatic-*-py3-none-any.whl dist/borgmatic-*-py3-none-any.whl.asc
+twine upload -r pypi --username __token__ dist/borgmatic-*.tar.gz
+twine upload -r pypi --username __token__ dist/borgmatic-*-py3-none-any.whl
 
 # Set release changelogs on projects.torsion.org and GitHub.
 release_changelog="$(cat NEWS | sed '/^$/q' | grep -v '^\S')"
@@ -50,5 +48,5 @@ curl --silent --request POST \
     --header "Content-Type: application/json" \
     --data "{\"body\": \"$escaped_release_changelog\", \"draft\": false, \"name\": \"borgmatic $version\", \"prerelease\": false, \"tag_name\": \"$version\"}"
 
-github-release create --token="$github_token" --owner=witten --repo=borgmatic --tag="$version" \
+github-release create --token="$github_token" --owner=witten --repo=borgmatic --tag="$version" --target_commit="main" \
     --name="borgmatic $version" --body="$release_changelog"

scripts/run-full-tests

@@ -10,7 +10,7 @@
 
 set -e
 
-if [ -z "$TEST_CONTAINER" ] ; then
+if [ -z "$TEST_CONTAINER" ]; then
     echo "This script is designed to work inside a test container and is not intended to"
     echo "be run manually. If you're trying to run borgmatic's end-to-end tests, execute"
     echo "scripts/run-end-to-end-dev-tests instead."
@@ -18,14 +18,14 @@ if [ -z "$TEST_CONTAINER" ] ; then
 fi
 
 apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client mongodb-tools \
-    py3-ruamel.yaml py3-ruamel.yaml.clib bash sqlite
+    py3-ruamel.yaml py3-ruamel.yaml.clib bash sqlite fish
 # If certain dependencies of black are available in this version of Alpine, install them.
 apk add --no-cache py3-typed-ast py3-regex || true
 python3 -m pip install --no-cache --upgrade pip==22.2.2 setuptools==64.0.1
 pip3 install --ignore-installed tox==3.25.1
 export COVERAGE_FILE=/tmp/.coverage
 
-if [ "$1" != "--end-to-end-only" ] ; then
+if [ "$1" != "--end-to-end-only" ]; then
     tox --workdir /tmp/.tox --sitepackages
 fi
 

setup.cfg

@@ -4,22 +4,23 @@ description_file=README.md
 [tool:pytest]
 testpaths = tests
 addopts = --cov-report term-missing:skip-covered --cov=borgmatic --ignore=tests/end-to-end
-filterwarnings = 
-    ignore:Deprecated call to `pkg_resources.declare_namespace\('ruamel'\)`.*:DeprecationWarning
 
 [flake8]
-ignore = E501,W503
+max-line-length = 100
+extend-ignore = E203,E501,W503
 exclude = *.*/*
 multiline-quotes = '''
 docstring-quotes = '''
 
 [tool:isort]
-force_single_line = False
-include_trailing_comma = True
+profile=black
 known_first_party = borgmatic
 line_length = 100
-multi_line_output = 3
 skip = .tox
 
 [codespell]
 skip = .git,.tox,build
+
+[pycodestyle]
+ignore = E203
+max_line_length = 100

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.7.12'
+VERSION = '1.7.15'
 
 
 setup(
@@ -23,7 +23,6 @@ setup(
     entry_points={
         'console_scripts': [
             'borgmatic = borgmatic.commands.borgmatic:main',
-            'upgrade-borgmatic-config = borgmatic.commands.convert_config:main',
             'generate-borgmatic-config = borgmatic.commands.generate_config:main',
             'validate-borgmatic-config = borgmatic.commands.validate_config:main',
         ]
@@ -32,6 +31,7 @@ setup(
     install_requires=(
         'colorama>=0.4.1,<0.5',
         'jsonschema',
+        'packaging',
         'requests',
         'ruamel.yaml>0.15.0,<0.18.0',
         'setuptools',

test_requirements.txt

@@ -1,27 +1,33 @@
 appdirs==1.4.4; python_version >= '3.8'
-attrs==20.3.0; python_version >= '3.8'
-black==19.10b0; python_version >= '3.8'
-click==7.1.2; python_version >= '3.8'
+attrs==22.2.0; python_version >= '3.8'
+black==23.3.0; python_version >= '3.8'
+chardet==5.1.0
+click==8.1.3; python_version >= '3.8'
 codespell==2.2.4
-colorama==0.4.4
-coverage==5.3
-flake8==4.0.1
+colorama==0.4.6
+coverage==7.2.3
+flake8==6.0.0
 flake8-quotes==3.3.2
 flake8-use-fstring==1.4
 flake8-variables-names==0.0.5
-flexmock==0.10.4
-isort==5.9.1
-mccabe==0.6.1
-pluggy==0.13.1
-pathspec==0.8.1; python_version >= '3.8'
-py==1.10.0
-pycodestyle==2.8.0
-pyflakes==2.4.0
-jsonschema==3.2.0
-pytest==7.2.0
+flexmock==0.11.3
+idna==3.4
+importlib_metadata==6.3.0; python_version < '3.8'
+isort==5.12.0
+mccabe==0.7.0
+packaging==23.1
+pluggy==1.0.0
+pathspec==0.11.1; python_version >= '3.8'
+py==1.11.0
+pycodestyle==2.10.0
+pyflakes==3.0.1
+jsonschema==4.17.3
+pytest==7.3.0
 pytest-cov==4.0.0
 regex; python_version >= '3.8'
-requests==2.25.0
+requests==2.31.0
 ruamel.yaml>0.15.0,<0.18.0
 toml==0.10.2; python_version >= '3.8'
 typed-ast; python_version >= '3.8'
+typing-extensions==4.5.0; python_version < '3.8'
+zipp==3.15.0; python_version < '3.8'

tests/end-to-end/docker-compose.yaml

@@ -5,16 +5,35 @@ services:
     environment:
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
+  postgresql2:
+    image: docker.io/postgres:13.1-alpine
+    environment:
+      POSTGRES_PASSWORD: test2
+      POSTGRES_DB: test
+      POSTGRES_USER: postgres2
+    command: docker-entrypoint.sh -p 5433
   mysql:
     image: docker.io/mariadb:10.5
     environment:
       MYSQL_ROOT_PASSWORD: test
       MYSQL_DATABASE: test
+  mysql2:
+    image: docker.io/mariadb:10.5
+    environment:
+      MYSQL_ROOT_PASSWORD: test2
+      MYSQL_DATABASE: test
+    command: docker-entrypoint.sh --port=3307
   mongodb:
     image: docker.io/mongo:5.0.5
     environment:
       MONGO_INITDB_ROOT_USERNAME: root
       MONGO_INITDB_ROOT_PASSWORD: test
+  mongodb2:
+    image: docker.io/mongo:5.0.5
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: root2
+      MONGO_INITDB_ROOT_PASSWORD: test2
+    command: docker-entrypoint.sh --port=27018
   tests:
     image: docker.io/alpine:3.13
     environment:
@@ -23,11 +42,15 @@ services:
       - "../..:/app:ro"
     tmpfs:
       - "/app/borgmatic.egg-info"
+      - "/app/build"
     tty: true
     working_dir: /app
     entrypoint: /app/scripts/run-full-tests
     command: --end-to-end-only
     depends_on:
       - postgresql
+      - postgresql2
       - mysql
+      - mysql2
       - mongodb
+      - mongodb2

tests/end-to-end/test_borgmatic.py

@@ -12,7 +12,7 @@ def generate_configuration(config_path, repository_path):
     to work for testing (including injecting the given repository path and tacking on an encryption
     passphrase).
     '''
-    subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' '))
+    subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' '))
     config = (
         open(config_path)
         .read()
@@ -44,7 +44,7 @@ def test_borgmatic_command():
         generate_configuration(config_path, repository_path)
 
         subprocess.check_call(
-            f'borgmatic -v 2 --config {config_path} init --encryption repokey'.split(' ')
+            f'borgmatic -v 2 --config {config_path} rcreate --encryption repokey'.split(' ')
         )
 
         # Run borgmatic to generate a backup archive, and then list it to make sure it exists.

tests/end-to-end/test_completion.py

@@ -3,3 +3,7 @@ import subprocess
 
 def test_bash_completion_runs_without_error():
     subprocess.check_call('borgmatic --bash-completion | bash', shell=True)
+
+
+def test_fish_completion_runs_without_error():
+    subprocess.check_call('borgmatic --fish-completion | fish', shell=True)

tests/end-to-end/test_database.py

@@ -82,6 +82,108 @@ hooks:
         config_file.write(config)
 
 
+def write_custom_restore_configuration(
+    source_directory,
+    config_path,
+    repository_path,
+    borgmatic_source_directory,
+    postgresql_dump_format='custom',
+    mongodb_dump_format='archive',
+):
+    '''
+    Write out borgmatic configuration into a file at the config path. Set the options so as to work
+    for testing with custom restore options. This includes a custom restore_hostname, restore_port,
+    restore_username, restore_password and restore_path.
+    '''
+    config = f'''
+location:
+    source_directories:
+        - {source_directory}
+    repositories:
+        - {repository_path}
+    borgmatic_source_directory: {borgmatic_source_directory}
+
+storage:
+    encryption_passphrase: "test"
+
+hooks:
+    postgresql_databases:
+        - name: test
+          hostname: postgresql
+          username: postgres
+          password: test
+          format: {postgresql_dump_format}
+          restore_hostname: postgresql2
+          restore_port: 5433
+          restore_username: postgres2
+          restore_password: test2
+    mysql_databases:
+        - name: test
+          hostname: mysql
+          username: root
+          password: test
+          restore_hostname: mysql2
+          restore_port: 3307
+          restore_username: root
+          restore_password: test2
+    mongodb_databases:
+        - name: test
+          hostname: mongodb
+          username: root
+          password: test
+          authentication_database: admin
+          format: {mongodb_dump_format}
+          restore_hostname: mongodb2
+          restore_port: 27018
+          restore_username: root2
+          restore_password: test2
+    sqlite_databases:
+        - name: sqlite_test
+          path: /tmp/sqlite_test.db
+          restore_path: /tmp/sqlite_test2.db
+'''
+
+    with open(config_path, 'w') as config_file:
+        config_file.write(config)
+
+
+def write_simple_custom_restore_configuration(
+    source_directory,
+    config_path,
+    repository_path,
+    borgmatic_source_directory,
+    postgresql_dump_format='custom',
+):
+    '''
+    Write out borgmatic configuration into a file at the config path. Set the options so as to work
+    for testing with custom restore options, but this time using CLI arguments. This includes a
+    custom restore_hostname, restore_port, restore_username and restore_password as we only test
+    these options for PostgreSQL.
+    '''
+    config = f'''
+location:
+    source_directories:
+        - {source_directory}
+    repositories:
+        - {repository_path}
+    borgmatic_source_directory: {borgmatic_source_directory}
+
+storage:
+    encryption_passphrase: "test"
+
+hooks:
+    postgresql_databases:
+        - name: test
+          hostname: postgresql
+          username: postgres
+          password: test
+          format: {postgresql_dump_format}
+'''
+
+    with open(config_path, 'w') as config_file:
+        config_file.write(config)
+
+
 def test_database_dump_and_restore():
     # Create a Borg repository.
     temporary_directory = tempfile.mkdtemp()
@@ -100,7 +202,7 @@ def test_database_dump_and_restore():
         )
 
         subprocess.check_call(
-            ['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey']
+            ['borgmatic', '-v', '2', '--config', config_path, 'rcreate', '--encryption', 'repokey']
         )
 
         # Run borgmatic to generate a backup archive including a database dump.
@@ -118,7 +220,104 @@ def test_database_dump_and_restore():
 
         # Restore the database from the archive.
         subprocess.check_call(
-            ['borgmatic', '--config', config_path, 'restore', '--archive', archive_name]
+            ['borgmatic', '-v', '2', '--config', config_path, 'restore', '--archive', archive_name]
+        )
+    finally:
+        os.chdir(original_working_directory)
+        shutil.rmtree(temporary_directory)
+
+
+def test_database_dump_and_restore_with_restore_cli_arguments():
+    # Create a Borg repository.
+    temporary_directory = tempfile.mkdtemp()
+    repository_path = os.path.join(temporary_directory, 'test.borg')
+    borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic')
+
+    original_working_directory = os.getcwd()
+
+    try:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+        write_simple_custom_restore_configuration(
+            temporary_directory, config_path, repository_path, borgmatic_source_directory
+        )
+
+        subprocess.check_call(
+            ['borgmatic', '-v', '2', '--config', config_path, 'rcreate', '--encryption', 'repokey']
+        )
+
+        # Run borgmatic to generate a backup archive including a database dump.
+        subprocess.check_call(['borgmatic', 'create', '--config', config_path, '-v', '2'])
+
+        # Get the created archive name.
+        output = subprocess.check_output(
+            ['borgmatic', '--config', config_path, 'list', '--json']
+        ).decode(sys.stdout.encoding)
+        parsed_output = json.loads(output)
+
+        assert len(parsed_output) == 1
+        assert len(parsed_output[0]['archives']) == 1
+        archive_name = parsed_output[0]['archives'][0]['archive']
+
+        # Restore the database from the archive.
+        subprocess.check_call(
+            [
+                'borgmatic',
+                '-v',
+                '2',
+                '--config',
+                config_path,
+                'restore',
+                '--archive',
+                archive_name,
+                '--hostname',
+                'postgresql2',
+                '--port',
+                '5433',
+                '--username',
+                'postgres2',
+                '--password',
+                'test2',
+            ]
+        )
+    finally:
+        os.chdir(original_working_directory)
+        shutil.rmtree(temporary_directory)
+
+
+def test_database_dump_and_restore_with_restore_configuration_options():
+    # Create a Borg repository.
+    temporary_directory = tempfile.mkdtemp()
+    repository_path = os.path.join(temporary_directory, 'test.borg')
+    borgmatic_source_directory = os.path.join(temporary_directory, '.borgmatic')
+
+    original_working_directory = os.getcwd()
+
+    try:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+        write_custom_restore_configuration(
+            temporary_directory, config_path, repository_path, borgmatic_source_directory
+        )
+
+        subprocess.check_call(
+            ['borgmatic', '-v', '2', '--config', config_path, 'rcreate', '--encryption', 'repokey']
+        )
+
+        # Run borgmatic to generate a backup archive including a database dump.
+        subprocess.check_call(['borgmatic', 'create', '--config', config_path, '-v', '2'])
+
+        # Get the created archive name.
+        output = subprocess.check_output(
+            ['borgmatic', '--config', config_path, 'list', '--json']
+        ).decode(sys.stdout.encoding)
+        parsed_output = json.loads(output)
+
+        assert len(parsed_output) == 1
+        assert len(parsed_output[0]['archives']) == 1
+        archive_name = parsed_output[0]['archives'][0]['archive']
+
+        # Restore the database from the archive.
+        subprocess.check_call(
+            ['borgmatic', '-v', '2', '--config', config_path, 'restore', '--archive', archive_name]
         )
     finally:
         os.chdir(original_working_directory)
@@ -145,7 +344,7 @@ def test_database_dump_and_restore_with_directory_format():
         )
 
         subprocess.check_call(
-            ['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey']
+            ['borgmatic', '-v', '2', '--config', config_path, 'rcreate', '--encryption', 'repokey']
         )
 
         # Run borgmatic to generate a backup archive including a database dump.
@@ -175,7 +374,7 @@ def test_database_dump_with_error_causes_borgmatic_to_exit():
         )
 
         subprocess.check_call(
-            ['borgmatic', '-v', '2', '--config', config_path, 'init', '--encryption', 'repokey']
+            ['borgmatic', '-v', '2', '--config', config_path, 'rcreate', '--encryption', 'repokey']
         )
 
         # Run borgmatic with a config override such that the database dump fails.

tests/end-to-end/test_dev_parity_with_build_server.py

@@ -0,0 +1,38 @@
+import ruamel.yaml
+
+
+def test_dev_docker_compose_has_same_services_as_build_server_configuration():
+    '''
+    The end-to-end test configuration for local development and the build server's test
+    configuration use two different mechanisms for configuring and spinning up "services"—the
+    database containers upon which the end-to-end tests are reliant. The dev configuration uses
+    Docker Compose, while the Drone build server configuration uses its own similar-but-different
+    configuration file format.
+
+    Therefore, to ensure dev-build parity, these tests assert that the services are the same across
+    the dev and build configurations. This includes service name, container image, environment
+    variables, and commands.
+
+    This test only compares services and does not assert anything else about the respective testing
+    environments.
+    '''
+    yaml = ruamel.yaml.YAML(typ='safe')
+    dev_services = {
+        name: service
+        for name, service in yaml.load(open('tests/end-to-end/docker-compose.yaml').read())[
+            'services'
+        ].items()
+        if name != 'tests'
+    }
+    build_server_services = tuple(yaml.load_all(open('.drone.yml').read()))[0]['services']
+
+    assert len(dev_services) == len(build_server_services)
+
+    for build_service in build_server_services:
+        dev_service = dev_services[build_service['name']]
+        assert dev_service['image'] == build_service['image']
+        assert dev_service['environment'] == build_service['environment']
+
+        if 'command' in dev_service or 'commands' in build_service:
+            assert len(build_service['commands']) <= 1
+            assert dev_service['command'] == build_service['commands'][0]

tests/end-to-end/test_generate_config.py

@@ -8,9 +8,9 @@ def test_generate_borgmatic_config_with_merging_succeeds():
         config_path = os.path.join(temporary_directory, 'test.yaml')
         new_config_path = os.path.join(temporary_directory, 'new.yaml')
 
-        subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' '))
+        subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' '))
         subprocess.check_call(
-            f'generate-borgmatic-config --source {config_path} --destination {new_config_path}'.split(
+            f'borgmatic config generate --source {config_path} --destination {new_config_path}'.split(
                 ' '
             )
         )

tests/end-to-end/test_override.py

@@ -10,7 +10,7 @@ def generate_configuration(config_path, repository_path):
     to work for testing (including injecting the given repository path and tacking on an encryption
     passphrase).
     '''
-    subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' '))
+    subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' '))
     config = (
         open(config_path)
         .read()
@@ -39,7 +39,7 @@ def test_override_get_normalized():
         generate_configuration(config_path, repository_path)
 
         subprocess.check_call(
-            f'borgmatic -v 2 --config {config_path} init --encryption repokey'.split(' ')
+            f'borgmatic -v 2 --config {config_path} rcreate --encryption repokey'.split(' ')
         )
 
         # Run borgmatic with an override structured for an outdated config file format. If

tests/end-to-end/test_validate_config.py

@@ -8,7 +8,7 @@ def test_validate_config_command_with_valid_configuration_succeeds():
     with tempfile.TemporaryDirectory() as temporary_directory:
         config_path = os.path.join(temporary_directory, 'test.yaml')
 
-        subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' '))
+        subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' '))
         exit_code = subprocess.call(f'validate-borgmatic-config --config {config_path}'.split(' '))
 
         assert exit_code == 0
@@ -18,7 +18,7 @@ def test_validate_config_command_with_invalid_configuration_fails():
     with tempfile.TemporaryDirectory() as temporary_directory:
         config_path = os.path.join(temporary_directory, 'test.yaml')
 
-        subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' '))
+        subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' '))
         config = open(config_path).read().replace('keep_daily: 7', 'keep_daily: "7"')
         config_file = open(config_path, 'w')
         config_file.write(config)
@@ -33,7 +33,7 @@ def test_validate_config_command_with_show_flag_displays_configuration():
     with tempfile.TemporaryDirectory() as temporary_directory:
         config_path = os.path.join(temporary_directory, 'test.yaml')
 
-        subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' '))
+        subprocess.check_call(f'borgmatic config generate --destination {config_path}'.split(' '))
         output = subprocess.check_output(
             f'validate-borgmatic-config --config {config_path} --show'.split(' ')
         ).decode(sys.stdout.encoding)

tests/integration/actions/config/test_validate.py

@@ -0,0 +1,37 @@
+import argparse
+
+from flexmock import flexmock
+
+import borgmatic.logger
+from borgmatic.actions.config import validate as module
+
+
+def test_run_validate_with_show_renders_configurations():
+    log_lines = []
+    borgmatic.logger.add_custom_log_levels()
+
+    def fake_logger_answer(message):
+        log_lines.append(message)
+
+    flexmock(module.logger).should_receive('answer').replace_with(fake_logger_answer)
+
+    module.run_validate(argparse.Namespace(show=True), {'test.yaml': {'foo': {'bar': 'baz'}}})
+
+    assert log_lines == ['''foo:\n    bar: baz\n''']
+
+
+def test_run_validate_with_show_and_multiple_configs_renders_each():
+    log_lines = []
+    borgmatic.logger.add_custom_log_levels()
+
+    def fake_logger_answer(message):
+        log_lines.append(message)
+
+    flexmock(module.logger).should_receive('answer').replace_with(fake_logger_answer)
+
+    module.run_validate(
+        argparse.Namespace(show=True),
+        {'test.yaml': {'foo': {'bar': 'baz'}}, 'other.yaml': {'quux': 'value'}},
+    )
+
+    assert log_lines == ['---', 'foo:\n    bar: baz\n', '---', 'quux: value\n']