Bump version for release.

Updated the other unit tests, as I had to re-arrange argument order
Added an 'all' test for the postgres end-to-end test.

Ran black formatter on it all.

NEWS

@@ -1,3 +1,17 @@
+1.5.2
+ * #301: Fix MySQL restore error on "all" database dump by excluding system tables.
+ * Fix PostgreSQL restore error on "all" database dump by using "psql" for the restore instead of
+   "pg_restore".
+
+1.5.1
+ * #289: Tired of looking up the latest successful archive name in order to pass it to borgmatic
+   actions? Me too. Now you can specify "--archive latest" to all actions that accept an archive
+   flag.
+ * #290: Fix the "--stats" and "--files" flags so that they yield output at verbosity 0.
+ * Reduce the default verbosity of borgmatic logs sent to Healthchecks monitoring hook. Now, it's
+   warnings and errors only. You can increase the verbosity via the "--monitoring-verbosity" flag.
+ * Add security policy documentation in SECURITY.md.
+
 1.5.0
  * #245: Monitor backups with PagerDuty hook integration. See the documentation for more
    information: https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#pagerduty-hook

README.md

@@ -119,14 +119,22 @@ If you'd like to chat with borgmatic developers or users, head on over to the
 href="https://webchat.freenode.net/?channels=borgmatic">web chat</a> or a
 native <a href="irc://chat.freenode.net:6697">IRC client</a>.
 
-Other questions or comments? Contact <mailto:witten@torsion.org>.
+Also see the [security
+policy](https://torsion.org/borgmatic/docs/security-policy/) for any security
+issues.
+
+Other questions or comments? Contact
+[witten@torsion.org](mailto:witten@torsion.org).
 
 
 ### Contributing
 
 borgmatic is hosted at <https://torsion.org/borgmatic> with [source code
-available](https://projects.torsion.org/witten/borgmatic). It's also mirrored
-on [GitHub](https://github.com/witten/borgmatic) for convenience.
+available](https://projects.torsion.org/witten/borgmatic), and is also
+mirrored on [GitHub](https://github.com/witten/borgmatic) for convenience.
+
+borgmatic is licensed under the GNU General Public License version 3 or any
+later version.
 
 If you'd like to contribute to borgmatic development, please feel free to
 submit a [Pull Request](https://projects.torsion.org/witten/borgmatic/pulls)

SECURITY.md

@@ -0,0 +1,19 @@
+---
+title: Security policy
+permalink: security-policy/index.html
+---
+
+## Supported versions
+
+While we want to hear about security vulnerabilities in all versions of
+borgmatic, security fixes will only be made to the most recently released
+version. It's not practical for our small volunteer effort to maintain
+multiple different release branches and put out separate security patches for
+each.
+
+## Reporting a vulnerability
+
+If you find a security vulnerability, please [file a
+ticket](https://torsion.org/borgmatic/#issues) or [send email
+directly](mailto:witten@torsion.org) as appropriate. You should expect to hear
+back within a few days at most, and generally sooner.

borgmatic/borg/list.py

@@ -11,6 +11,42 @@ logger = logging.getLogger(__name__)
 BORG_EXCLUDE_CHECKPOINTS_GLOB = '*[0123456789]'
 
 
+def resolve_archive_name(repository, archive, storage_config, local_path='borg', remote_path=None):
+    '''
+    Given a local or remote repository path, an archive name, a storage config dict, a local Borg
+    path, and a remote Borg path, simply return the archive name. But if the archive name is
+    "latest", then instead introspect the repository for the latest successful (non-checkpoint)
+    archive, and return its name.
+
+    Raise ValueError if "latest" is given but there are no archives in the repository.
+    '''
+    if archive != "latest":
+        return archive
+
+    lock_wait = storage_config.get('lock_wait', None)
+
+    full_command = (
+        (local_path, 'list')
+        + (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+        + make_flags('remote-path', remote_path)
+        + make_flags('lock-wait', lock_wait)
+        + make_flags('glob-archives', BORG_EXCLUDE_CHECKPOINTS_GLOB)
+        + make_flags('last', 1)
+        + ('--short', repository)
+    )
+
+    output = execute_command(full_command, output_log_level=None, error_on_warnings=False)
+    try:
+        latest_archive = output.strip().splitlines()[-1]
+    except IndexError:
+        raise ValueError('No archives found in the repository')
+
+    logger.debug('{}: Latest archive is {}'.format(repository, latest_archive))
+
+    return latest_archive
+
+
 def list_archives(repository, storage_config, list_arguments, local_path='borg', remote_path=None):
     '''
     Given a local or remote repository path, a storage config dict, and the arguments to the list

borgmatic/commands/arguments.py

@@ -163,7 +163,7 @@ def parse_arguments(*unparsed_arguments):
         '--monitoring-verbosity',
         type=int,
         choices=range(-1, 3),
-        default=1,
+        default=0,
         help='Log verbose progress to monitoring integrations that support logging (from only errors to very verbose: -1, 0, 1, or 2)',
     )
     global_group.add_argument(
@@ -323,7 +323,9 @@ def parse_arguments(*unparsed_arguments):
         '--repository',
         help='Path of repository to extract, defaults to the configured repository if there is only one',
     )
-    extract_group.add_argument('--archive', help='Name of archive to extract', required=True)
+    extract_group.add_argument(
+        '--archive', help='Name of archive to extract (or "latest")', required=True
+    )
     extract_group.add_argument(
         '--path',
         '--restore-path',
@@ -361,7 +363,7 @@ def parse_arguments(*unparsed_arguments):
         '--repository',
         help='Path of repository to use, defaults to the configured repository if there is only one',
     )
-    mount_group.add_argument('--archive', help='Name of archive to mount')
+    mount_group.add_argument('--archive', help='Name of archive to mount (or "latest")')
     mount_group.add_argument(
         '--mount-point',
         metavar='PATH',
@@ -415,7 +417,9 @@ def parse_arguments(*unparsed_arguments):
         '--repository',
         help='Path of repository to restore from, defaults to the configured repository if there is only one',
     )
-    restore_group.add_argument('--archive', help='Name of archive to restore from', required=True)
+    restore_group.add_argument(
+        '--archive', help='Name of archive to restore from (or "latest")', required=True
+    )
     restore_group.add_argument(
         '--database',
         metavar='NAME',
@@ -446,7 +450,7 @@ def parse_arguments(*unparsed_arguments):
         '--repository',
         help='Path of repository to list, defaults to the configured repository if there is only one',
     )
-    list_group.add_argument('--archive', help='Name of archive to list')
+    list_group.add_argument('--archive', help='Name of archive to list (or "latest")')
     list_group.add_argument(
         '--path',
         metavar='PATH',
@@ -508,7 +512,7 @@ def parse_arguments(*unparsed_arguments):
         '--repository',
         help='Path of repository to show info for, defaults to the configured repository if there is only one',
     )
-    info_group.add_argument('--archive', help='Name of archive to show info for')
+    info_group.add_argument('--archive', help='Name of archive to show info for (or "latest")')
     info_group.add_argument(
         '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
     )

borgmatic/commands/borgmatic.py

@@ -1,4 +1,5 @@
 import collections
+import copy
 import json
 import logging
 import os
@@ -297,7 +298,9 @@ def run_actions(
             borg_extract.extract_archive(
                 global_arguments.dry_run,
                 repository,
-                arguments['extract'].archive,
+                borg_list.resolve_archive_name(
+                    repository, arguments['extract'].archive, storage, local_path, remote_path
+                ),
                 arguments['extract'].paths,
                 location,
                 storage,
@@ -319,7 +322,9 @@ def run_actions(
 
             borg_mount.mount_archive(
                 repository,
-                arguments['mount'].archive,
+                borg_list.resolve_archive_name(
+                    repository, arguments['mount'].archive, storage, local_path, remote_path
+                ),
                 arguments['mount'].mount_point,
                 arguments['mount'].paths,
                 arguments['mount'].foreground,
@@ -355,7 +360,9 @@ def run_actions(
             borg_extract.extract_archive(
                 global_arguments.dry_run,
                 repository,
-                arguments['restore'].archive,
+                borg_list.resolve_archive_name(
+                    repository, arguments['restore'].archive, storage, local_path, remote_path
+                ),
                 dump.convert_glob_patterns_to_borg_patterns(
                     dump.flatten_dump_patterns(dump_patterns, restore_names)
                 ),
@@ -395,12 +402,16 @@ def run_actions(
         if arguments['list'].repository is None or validate.repositories_match(
             repository, arguments['list'].repository
         ):
-            if not arguments['list'].json:
+            list_arguments = copy.copy(arguments['list'])
+            if not list_arguments.json:
                 logger.warning('{}: Listing archives'.format(repository))
+            list_arguments.archive = borg_list.resolve_archive_name(
+                repository, list_arguments.archive, storage, local_path, remote_path
+            )
             json_output = borg_list.list_archives(
                 repository,
                 storage,
-                list_arguments=arguments['list'],
+                list_arguments=list_arguments,
                 local_path=local_path,
                 remote_path=remote_path,
             )
@@ -410,12 +421,16 @@ def run_actions(
         if arguments['info'].repository is None or validate.repositories_match(
             repository, arguments['info'].repository
         ):
-            if not arguments['info'].json:
+            info_arguments = copy.copy(arguments['info'])
+            if not info_arguments.json:
                 logger.warning('{}: Displaying summary info for archives'.format(repository))
+            info_arguments.archive = borg_list.resolve_archive_name(
+                repository, info_arguments.archive, storage, local_path, remote_path
+            )
             json_output = borg_info.display_archives_info(
                 repository,
                 storage,
-                info_arguments=arguments['info'],
+                info_arguments=info_arguments,
                 local_path=local_path,
                 remote_path=remote_path,
             )

borgmatic/hooks/mysql.py

@@ -16,6 +16,43 @@ def make_dump_path(location_config):  # pragma: no cover
     )
 
 
+SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys')
+
+
+def database_names_to_dump(database, extra_environment, log_prefix, dry_run_label):
+    '''
+    Given a requested database name, return the corresponding sequence of database names to dump.
+    In the case of "all", query for the names of databases on the configured host and return them,
+    excluding any system databases that will cause problems during restore.
+    '''
+    requested_name = database['name']
+
+    if requested_name != 'all':
+        return (requested_name,)
+
+    show_command = (
+        ('mysql',)
+        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--port', str(database['port'])) if 'port' in database else ())
+        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--user', database['username']) if 'username' in database else ())
+        + ('--skip-column-names', '--batch')
+        + ('--execute', 'show schemas')
+    )
+    logger.debug(
+        '{}: Querying for "all" MySQL databases to dump{}'.format(log_prefix, dry_run_label)
+    )
+    show_output = execute_command(
+        show_command, output_log_level=None, extra_environment=extra_environment
+    )
+
+    return tuple(
+        show_name
+        for show_name in show_output.strip().splitlines()
+        if show_name not in SYSTEM_DATABASE_NAMES
+    )
+
+
 def dump_databases(databases, log_prefix, location_config, dry_run):
     '''
     Dump the given MySQL/MariaDB databases to disk. The databases are supplied as a sequence of
@@ -28,30 +65,37 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
     logger.info('{}: Dumping MySQL databases{}'.format(log_prefix, dry_run_label))
 
     for database in databases:
-        name = database['name']
+        requested_name = database['name']
         dump_filename = dump.make_database_dump_filename(
-            make_dump_path(location_config), name, database.get('hostname')
+            make_dump_path(location_config), requested_name, database.get('hostname')
         )
-        command = (
+        extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
+        dump_command_names = database_names_to_dump(
+            database, extra_environment, log_prefix, dry_run_label
+        )
+
+        dump_command = (
             ('mysqldump', '--add-drop-database')
             + (('--host', database['hostname']) if 'hostname' in database else ())
             + (('--port', str(database['port'])) if 'port' in database else ())
             + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
             + (('--user', database['username']) if 'username' in database else ())
             + (tuple(database['options'].split(' ')) if 'options' in database else ())
-            + (('--all-databases',) if name == 'all' else ('--databases', name))
+            + ('--databases',)
+            + dump_command_names
         )
-        extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
 
         logger.debug(
             '{}: Dumping MySQL database {} to {}{}'.format(
-                log_prefix, name, dump_filename, dry_run_label
+                log_prefix, requested_name, dump_filename, dry_run_label
             )
         )
         if not dry_run:
             os.makedirs(os.path.dirname(dump_filename), mode=0o700, exist_ok=True)
             execute_command(
-                command, output_file=open(dump_filename, 'w'), extra_environment=extra_environment
+                dump_command,
+                output_file=open(dump_filename, 'w'),
+                extra_environment=extra_environment,
             )
 
 

borgmatic/hooks/postgresql.py

@@ -34,7 +34,12 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
         )
         all_databases = bool(name == 'all')
         command = (
-            ('pg_dumpall' if all_databases else 'pg_dump', '--no-password', '--clean')
+            (
+                'pg_dumpall' if all_databases else 'pg_dump',
+                '--no-password',
+                '--clean',
+                '--if-exists',
+            )
             + ('--file', dump_filename)
             + (('--host', database['hostname']) if 'hostname' in database else ())
             + (('--port', str(database['port'])) if 'port' in database else ())
@@ -93,23 +98,28 @@ def restore_database_dumps(databases, log_prefix, location_config, dry_run):
         dump_filename = dump.make_database_dump_filename(
             make_dump_path(location_config), database['name'], database.get('hostname')
         )
-        restore_command = (
-            ('pg_restore', '--no-password', '--clean', '--if-exists', '--exit-on-error')
-            + (('--host', database['hostname']) if 'hostname' in database else ())
-            + (('--port', str(database['port'])) if 'port' in database else ())
-            + (('--username', database['username']) if 'username' in database else ())
-            + ('--dbname', database['name'])
-            + (dump_filename,)
-        )
-        extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
+        all_databases = bool(database['name'] == 'all')
         analyze_command = (
             ('psql', '--no-password', '--quiet')
             + (('--host', database['hostname']) if 'hostname' in database else ())
             + (('--port', str(database['port'])) if 'port' in database else ())
             + (('--username', database['username']) if 'username' in database else ())
-            + ('--dbname', database['name'])
+            + (('--dbname', database['name']) if not all_databases else ())
             + ('--command', 'ANALYZE')
         )
+        restore_command = (
+            ('psql' if all_databases else 'pg_restore', '--no-password')
+            + (
+                ('--if-exists', '--exit-on-error', '--clean', '--dbname', database['name'])
+                if not all_databases
+                else ()
+            )
+            + (('--host', database['hostname']) if 'hostname' in database else ())
+            + (('--port', str(database['port'])) if 'port' in database else ())
+            + (('--username', database['username']) if 'username' in database else ())
+            + (('-f', dump_filename) if all_databases else (dump_filename,))
+        )
+        extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
 
         logger.debug(
             '{}: Restoring PostgreSQL database {}{}'.format(

docs/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.7.4-alpine3.10 as borgmatic
+FROM python:3.8.1-alpine3.11 as borgmatic
 
 COPY . /app
 RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
@@ -7,7 +7,7 @@ RUN borgmatic --help > /command-line.txt \
            echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \
            && borgmatic "$action" --help >> /command-line.txt; done
 
-FROM node:12.10.0-alpine as html
+FROM node:13.7.0-alpine as html
 
 ARG ENVIRONMENT=production
 

docs/SECURITY.md

@@ -0,0 +1,19 @@
+---
+title: Security policy
+permalink: security-policy/index.html
+---
+
+## Supported versions
+
+While we want to hear about security vulnerabilities in all versions of
+borgmatic, security fixes will only be made to the most recently released
+version. It's not practical for our small volunteer effort to maintain
+multiple different release branches and put out separate security patches for
+each.
+
+## Reporting a vulnerability
+
+If you find a security vulnerability, please [file a
+ticket](https://torsion.org/borgmatic/#issues) or [send email
+directly](mailto:witten@torsion.org) as appropriate. You should expect to hear
+back within a few days at most, and generally sooner.

docs/how-to/backup-to-a-removable-drive-or-an-intermittent-server.md

@@ -10,7 +10,7 @@ buddy's sometimes-online server for that extra level of redundancy.
 
 But if you run borgmatic and your hard drive isn't plugged in, or your buddy's
 server is offline, then you'll get an annoying error message and the overall
-borgmatic run will fail (even if individual repositories complete just fine).
+borgmatic run will fail (even if individual repositories still complete).
 
 So what if you want borgmatic to swallow the error of a missing drive
 or an offline server, and continue trucking along? That's where the concept of
@@ -96,7 +96,7 @@ There are some caveats you should be aware of with this feature.
    a test to make sure that individual source directories are mounted and
    available. Use your imagination!
  * The soft failure feature also works for `before_prune`, `after_prune`,
-   `before_check`, and `after_check` hooks. However it is not implemented for
+   `before_check`, and `after_check` hooks. But it is not implemented for
    `before_everything` or `after_everything`.
 
 ## Related documentation

docs/how-to/backup-your-databases.md

@@ -112,6 +112,12 @@ borgmatic restore --archive host-2019-01-02T04:06:07.080910
 
 (No borgmatic `restore` action? Upgrade borgmatic!)
 
+With newer versions of borgmatic, you can simplify this to:
+
+```bash
+borgmatic restore --archive latest
+```
+
 The `--archive` value is the name of the archive to restore from. This
 restores all databases dumps that borgmatic originally backed up to that
 archive.

docs/how-to/deal-with-very-large-backups.md

@@ -7,7 +7,7 @@ Borg itself is great for efficiently de-duplicating data across successive
 backup archives, even when dealing with very large repositories. But you may
 find that while borgmatic's default mode of "prune, create, and check" works
 well on small repositories, it's not so great on larger ones. That's because
-running the default consistency checks takes a long time on large
+running the default pruning and consistency checks take a long time on large
 repositories.
 
 ### A la carte actions
@@ -27,9 +27,18 @@ borgmatic check
 
 You can run with only one of these actions provided, or you can mix and match
 any number of them in a single borgmatic run. This supports approaches like
-making backups with `create` on a frequent schedule, while only running
-expensive consistency checks with `check` on a much less frequent basis from
-a separate cron job.
+skipping certain actions while running others. For instance, this skips
+`prune` and only runs `create` and `check`:
+
+```bash
+borgmatic create check
+```
+
+Or, you can make backups with `create` on a frequent schedule (e.g. with
+`borgmatic create` called from one cron job), while only running expensive
+consistency checks with `check` on a much less frequent basis (e.g. with
+`borgmatic check` called from a separate cron job).
+
 
 ### Consistency check configuration
 

docs/how-to/extract-a-backup.md

@@ -31,6 +31,12 @@ borgmatic extract --archive host-2019-01-02T04:06:07.080910
 (No borgmatic `extract` action? Try the old-style `--extract`, or upgrade
 borgmatic!)
 
+With newer versions of borgmatic, you can simplify this to:
+
+```bash
+borgmatic extract --archive latest
+```
+
 The `--archive` value is the name of the archive to extract. This extracts the
 entire contents of the archive to the current directory, so make sure you're
 in the right place before running the command.
@@ -106,6 +112,12 @@ Omit the `--archive` flag to mount all archives (lazy-loaded):
 borgmatic mount --mount-point /mnt
 ```
 
+Or use the "latest" value for the archive to mount the latest successful archive:
+
+```bash
+borgmatic mount --archive latest --mount-point /mnt
+```
+
 If you'd like to restrict the mounted filesystem to only particular paths from
 your archive, use the `--path` flag, similar to the `extract` action above.
 For instance:

docs/how-to/monitor-your-backups.md

@@ -203,12 +203,21 @@ or it doesn't hear from borgmatic for a certain period of time.
 
 ## PagerDuty hook
 
-[PagerDuty](https://cronhub.io/) provides incident monitoring and alerting,
-and borgmatic has built-in integration with it. Once you create a PagerDuty
-account and <a
+In case you're new here: [borgmatic](https://torsion.org/borgmatic/) is
+simple, configuration-driven backup software for servers and workstations,
+powered by [Borg Backup](https://www.borgbackup.org/).
+
+[PagerDuty](https://www.pagerduty.com/) provides incident monitoring and
+alerting. borgmatic has built-in integration that can notify you via PagerDuty
+as soon as a backup fails, so you can make sure your backups keep working.
+
+First, create a PagerDuty account and <a
 href="https://support.pagerduty.com/docs/services-and-integrations">service</a>
-on their site, all you need to do is configure borgmatic with the unique
-"Integration Key" for your service. Here's an example:
+on their site. On the service, add an integration and set the Integration Type
+to "borgmatic".
+
+Then, configure borgmatic with the unique "Integration Key" for your service.
+Here's an example:
 
 
 ```yaml
@@ -226,6 +235,9 @@ You can configure PagerDuty to notify you by a [variety of
 mechanisms](https://support.pagerduty.com/docs/notifications) when backups
 fail.
 
+If you have any issues with the integration, [please contact
+us](https://torsion.org/borgmatic/#support-and-contributing).
+
 
 ## Scripting borgmatic
 
@@ -261,6 +273,18 @@ multiple different hosts into a single repository, then you'll need to get
 fancier with your archive listing. See `borg list --help` for more flags.
 
 
+### Latest backups
+
+All borgmatic actions that accept an "--archive" flag allow you to specify an
+archive name of "latest". This lets you get the latest successful archive
+without having to first run "borgmatic list" manually, which can be handy in
+automated scripts. Here's an example:
+
+```bash
+borgmatic info --archive latest
+```
+
+
 ## Related documentation
 
  * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)

docs/how-to/set-up-backups.md

@@ -3,6 +3,11 @@ title: How to set up backups with borgmatic
 ---
 ## Installation
 
+Many users need to backup system files that require privileged access, so
+these instructions install and run borgmatic as root. If you don't need to
+backup such files, then you are welcome to install and run borgmatic as a
+non-root user.
+
 First, [install
 Borg](https://borgbackup.readthedocs.io/en/stable/installation.html), at least
 version 1.1.
@@ -39,6 +44,7 @@ borgmatic:
  * [Fedora official](https://bodhi.fedoraproject.org/updates/?search=borgmatic)
  * [Fedora unofficial](https://copr.fedorainfracloud.org/coprs/heffer/borgmatic/)
  * [Arch Linux](https://www.archlinux.org/packages/community/any/borgmatic/)
+ * [Alpine Linux](https://pkgs.alpinelinux.org/packages?name=borgmatic)
  * [OpenBSD](http://ports.su/sysutils/borgmatic)
  * [openSUSE](https://software.opensuse.org/package/borgmatic)
  * [stand-alone binary](https://github.com/cmarquardt/borgmatic-binary)
@@ -132,7 +138,7 @@ this step if you already have a Borg repository.) To create a repository, run
 a command like the following:
 
 ```bash
-borgmatic init --encryption repokey
+sudo borgmatic init --encryption repokey
 ```
 
 (No borgmatic `init` action? Try the old-style `--init` flag, or upgrade
@@ -164,7 +170,7 @@ good idea to test that borgmatic is working. So to run borgmatic and start a
 backup, you can invoke it like this:
 
 ```bash
-borgmatic --verbosity 1
+sudo borgmatic --verbosity 1
 ```
 
 By default, this will also prune any old backups as per the configured

scripts/release

@@ -23,9 +23,10 @@ git push github $version
 rm -fr dist
 python3 setup.py bdist_wheel
 python3 setup.py sdist
-gpg --detach-sign --armor dist/*
-twine upload -r pypi dist/borgmatic-*.tar.gz
-twine upload -r pypi dist/borgmatic-*-py3-none-any.whl
+gpg --detach-sign --armor dist/borgmatic-*.tar.gz
+gpg --detach-sign --armor dist/borgmatic-*-py3-none-any.whl
+twine upload -r pypi dist/borgmatic-*.tar.gz dist/borgmatic-*.tar.gz.asc
+twine upload -r pypi dist/borgmatic-*-py3-none-any.whl borgmatic-*-py3-none-any.whl.asc
 
 # Set release changelogs on projects.torsion.org and GitHub.
 release_changelog="$(cat NEWS | sed '/^$/q' | grep -v '^\S')"

scripts/run-full-dev-tests

@@ -11,4 +11,4 @@
 set -e
 
 docker-compose --file tests/end-to-end/docker-compose.yaml up --force-recreate \
-    --abort-on-container-exit
+    --renew-anon-volumes --abort-on-container-exit

scripts/run-full-tests

@@ -10,8 +10,8 @@
 
 set -e
 
-python -m pip install --upgrade pip==19.3.1
-pip install tox==3.14.1
+python -m pip install --upgrade pip==20.0.2
+pip install tox==3.14.3
 export COVERAGE_FILE=/tmp/.coverage
 tox --workdir /tmp/.tox
 apk add --no-cache borgbackup postgresql-client mariadb-client

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.5.0'
+VERSION = '1.5.2'
 
 
 setup(

tests/end-to-end/docker-compose.yaml

@@ -1,17 +1,17 @@
 version: '3'
 services:
   postgresql:
-    image: postgres:11.6-alpine
+    image: postgres:12.2-alpine
     environment:
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
   mysql:
-    image: mariadb:10.4
+    image: mariadb:10.5
     environment:
       MYSQL_ROOT_PASSWORD: test
       MYSQL_DATABASE: test
   tests:
-    image: python:3.7-alpine3.10
+    image: python:3.8-alpine3.11
     volumes:
       - "../..:/app:ro"
     tmpfs:

tests/end-to-end/test_database.py

@@ -29,11 +29,19 @@ hooks:
           hostname: postgresql
           username: postgres
           password: test
+        - name: all
+          hostname: postgresql
+          username: postgres
+          password: test
     mysql_databases:
         - name: test
           hostname: mysql
           username: root
           password: test
+        - name: all
+          hostname: mysql
+          username: root
+          password: test
 '''.format(
         config_path, repository_path, borgmatic_source_directory
     )

tests/unit/borg/test_list.py

@@ -7,6 +7,110 @@ from borgmatic.borg import list as module
 
 from ..test_verbosity import insert_logging_mock
 
+BORG_LIST_LATEST_ARGUMENTS = (
+    '--glob-archives',
+    module.BORG_EXCLUDE_CHECKPOINTS_GLOB,
+    '--last',
+    '1',
+    '--short',
+    'repo',
+)
+
+
+def test_resolve_archive_name_passes_through_non_latest_archive_name():
+    archive = 'myhost-2030-01-01T14:41:17.647620'
+
+    assert module.resolve_archive_name('repo', archive, storage_config={}) == archive
+
+
+def test_resolve_archive_name_calls_borg_with_parameters():
+    expected_archive = 'archive-name'
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
+        output_log_level=None,
+        error_on_warnings=False,
+    ).and_return(expected_archive + '\n')
+
+    assert module.resolve_archive_name('repo', 'latest', storage_config={}) == expected_archive
+
+
+def test_resolve_archive_name_with_log_info_calls_borg_with_info_parameter():
+    expected_archive = 'archive-name'
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'list', '--info') + BORG_LIST_LATEST_ARGUMENTS,
+        output_log_level=None,
+        error_on_warnings=False,
+    ).and_return(expected_archive + '\n')
+    insert_logging_mock(logging.INFO)
+
+    assert module.resolve_archive_name('repo', 'latest', storage_config={}) == expected_archive
+
+
+def test_resolve_archive_name_with_log_debug_calls_borg_with_debug_parameter():
+    expected_archive = 'archive-name'
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'list', '--debug', '--show-rc') + BORG_LIST_LATEST_ARGUMENTS,
+        output_log_level=None,
+        error_on_warnings=False,
+    ).and_return(expected_archive + '\n')
+    insert_logging_mock(logging.DEBUG)
+
+    assert module.resolve_archive_name('repo', 'latest', storage_config={}) == expected_archive
+
+
+def test_resolve_archive_name_with_local_path_calls_borg_via_local_path():
+    expected_archive = 'archive-name'
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg1', 'list') + BORG_LIST_LATEST_ARGUMENTS,
+        output_log_level=None,
+        error_on_warnings=False,
+    ).and_return(expected_archive + '\n')
+
+    assert (
+        module.resolve_archive_name('repo', 'latest', storage_config={}, local_path='borg1')
+        == expected_archive
+    )
+
+
+def test_resolve_archive_name_with_remote_path_calls_borg_with_remote_path_parameters():
+    expected_archive = 'archive-name'
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'list', '--remote-path', 'borg1') + BORG_LIST_LATEST_ARGUMENTS,
+        output_log_level=None,
+        error_on_warnings=False,
+    ).and_return(expected_archive + '\n')
+
+    assert (
+        module.resolve_archive_name('repo', 'latest', storage_config={}, remote_path='borg1')
+        == expected_archive
+    )
+
+
+def test_resolve_archive_name_without_archives_raises():
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'list') + BORG_LIST_LATEST_ARGUMENTS,
+        output_log_level=None,
+        error_on_warnings=False,
+    ).and_return('')
+
+    with pytest.raises(ValueError):
+        module.resolve_archive_name('repo', 'latest', storage_config={})
+
+
+def test_resolve_archive_name_with_lock_wait_calls_borg_with_lock_wait_parameters():
+    expected_archive = 'archive-name'
+
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'list', '--lock-wait', 'okay') + BORG_LIST_LATEST_ARGUMENTS,
+        output_log_level=None,
+        error_on_warnings=False,
+    ).and_return(expected_archive + '\n')
+
+    assert (
+        module.resolve_archive_name('repo', 'latest', storage_config={'lock_wait': 'okay'})
+        == expected_archive
+    )
+
 
 def test_list_archives_calls_borg_with_parameters():
     flexmock(module).should_receive('execute_command').with_args(

tests/unit/hooks/test_mysql.py

@@ -5,6 +5,35 @@ from flexmock import flexmock
 from borgmatic.hooks import mysql as module
 
 
+def test_database_names_to_dump_passes_through_name():
+    extra_environment = flexmock()
+    log_prefix = ''
+    dry_run_label = ''
+
+    names = module.database_names_to_dump(
+        {'name': 'foo'}, extra_environment, log_prefix, dry_run_label
+    )
+
+    assert names == ('foo',)
+
+
+def test_database_names_to_dump_queries_mysql_for_database_names():
+    extra_environment = flexmock()
+    log_prefix = ''
+    dry_run_label = ''
+    flexmock(module).should_receive('execute_command').with_args(
+        ('mysql', '--skip-column-names', '--batch', '--execute', 'show schemas'),
+        output_log_level=None,
+        extra_environment=extra_environment,
+    ).and_return('foo\nbar\nmysql\n').once()
+
+    names = module.database_names_to_dump(
+        {'name': 'all'}, extra_environment, log_prefix, dry_run_label
+    )
+
+    assert names == ('foo', 'bar')
+
+
 def test_dump_databases_runs_mysqldump_for_each_database():
     databases = [{'name': 'foo'}, {'name': 'bar'}]
     output_file = flexmock()
@@ -12,6 +41,9 @@ def test_dump_databases_runs_mysqldump_for_each_database():
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     ).and_return('databases/localhost/bar')
+    flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
+        ('bar',)
+    )
     flexmock(module.os).should_receive('makedirs')
     flexmock(sys.modules['builtins']).should_receive('open').and_return(output_file)
 
@@ -31,6 +63,9 @@ def test_dump_databases_with_dry_run_skips_mysqldump():
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     ).and_return('databases/localhost/bar')
+    flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
+        ('bar',)
+    )
     flexmock(module.os).should_receive('makedirs').never()
     flexmock(module).should_receive('execute_command').never()
 
@@ -44,6 +79,7 @@ def test_dump_databases_runs_mysqldump_with_hostname_and_port():
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/database.example.org/foo'
     )
+    flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
     flexmock(module.os).should_receive('makedirs')
     flexmock(sys.modules['builtins']).should_receive('open').and_return(output_file)
 
@@ -74,6 +110,7 @@ def test_dump_databases_runs_mysqldump_with_username_and_password():
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     )
+    flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
     flexmock(module.os).should_receive('makedirs')
     flexmock(sys.modules['builtins']).should_receive('open').and_return(output_file)
 
@@ -93,6 +130,7 @@ def test_dump_databases_runs_mysqldump_with_options():
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     )
+    flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
     flexmock(module.os).should_receive('makedirs')
     flexmock(sys.modules['builtins']).should_receive('open').and_return(output_file)
 
@@ -112,11 +150,12 @@ def test_dump_databases_runs_mysqldump_for_all_databases():
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/all'
     )
+    flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar'))
     flexmock(module.os).should_receive('makedirs')
     flexmock(sys.modules['builtins']).should_receive('open').and_return(output_file)
 
     flexmock(module).should_receive('execute_command').with_args(
-        ('mysqldump', '--add-drop-database', '--all-databases'),
+        ('mysqldump', '--add-drop-database', '--databases', 'foo', 'bar'),
         output_file=output_file,
         extra_environment=None,
     ).once()

tests/unit/hooks/test_postgresql.py

@@ -17,6 +17,7 @@ def test_dump_databases_runs_pg_dump_for_each_database():
                 'pg_dump',
                 '--no-password',
                 '--clean',
+                '--if-exists',
                 '--file',
                 'databases/localhost/{}'.format(name),
                 '--format',
@@ -54,6 +55,7 @@ def test_dump_databases_runs_pg_dump_with_hostname_and_port():
             'pg_dump',
             '--no-password',
             '--clean',
+            '--if-exists',
             '--file',
             'databases/database.example.org/foo',
             '--host',
@@ -83,6 +85,7 @@ def test_dump_databases_runs_pg_dump_with_username_and_password():
             'pg_dump',
             '--no-password',
             '--clean',
+            '--if-exists',
             '--file',
             'databases/localhost/foo',
             '--username',
@@ -110,6 +113,7 @@ def test_dump_databases_runs_pg_dump_with_format():
             'pg_dump',
             '--no-password',
             '--clean',
+            '--if-exists',
             '--file',
             'databases/localhost/foo',
             '--format',
@@ -135,6 +139,7 @@ def test_dump_databases_runs_pg_dump_with_options():
             'pg_dump',
             '--no-password',
             '--clean',
+            '--if-exists',
             '--file',
             'databases/localhost/foo',
             '--format',
@@ -157,7 +162,14 @@ def test_dump_databases_runs_pg_dumpall_for_all_databases():
     flexmock(module.os).should_receive('makedirs')
 
     flexmock(module).should_receive('execute_command').with_args(
-        ('pg_dumpall', '--no-password', '--clean', '--file', 'databases/localhost/all'),
+        (
+            'pg_dumpall',
+            '--no-password',
+            '--clean',
+            '--if-exists',
+            '--file',
+            'databases/localhost/all',
+        ),
         extra_environment=None,
     ).once()
 
@@ -197,9 +209,9 @@ def test_restore_database_dumps_restores_each_database():
             (
                 'pg_restore',
                 '--no-password',
-                '--clean',
                 '--if-exists',
                 '--exit-on-error',
+                '--clean',
                 '--dbname',
                 name,
                 'databases/localhost/{}'.format(name),
@@ -225,15 +237,15 @@ def test_restore_database_dumps_runs_pg_restore_with_hostname_and_port():
         (
             'pg_restore',
             '--no-password',
-            '--clean',
             '--if-exists',
             '--exit-on-error',
+            '--clean',
+            '--dbname',
+            'foo',
             '--host',
             'database.example.org',
             '--port',
             '5433',
-            '--dbname',
-            'foo',
             'databases/localhost/foo',
         ),
         extra_environment=None,
@@ -269,13 +281,13 @@ def test_restore_database_dumps_runs_pg_restore_with_username_and_password():
         (
             'pg_restore',
             '--no-password',
-            '--clean',
             '--if-exists',
             '--exit-on-error',
-            '--username',
-            'postgres',
+            '--clean',
             '--dbname',
             'foo',
+            '--username',
+            'postgres',
             'databases/localhost/foo',
         ),
         extra_environment={'PGPASSWORD': 'trustsome1'},
@@ -296,3 +308,20 @@ def test_restore_database_dumps_runs_pg_restore_with_username_and_password():
     ).once()
 
     module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)
+
+
+def test_restore_database_dumps_runs_psql_for_all_database_dump():
+    databases = [{'name': 'all'}]
+    flexmock(module).should_receive('make_dump_path').and_return('')
+    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
+        'databases/localhost/all'
+    )
+
+    flexmock(module).should_receive('execute_command').with_args(
+        ('psql', '--no-password', '-f', 'databases/localhost/all'), extra_environment=None
+    ).once()
+    flexmock(module).should_receive('execute_command').with_args(
+        ('psql', '--no-password', '--quiet', '--command', 'ANALYZE'), extra_environment=None
+    ).once()
+
+    module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)