Bump version for release.

Updated the other unit tests, as I had to re-arrange argument order
Added an 'all' test for the postgres end-to-end test.

Ran black formatter on it all.

NEWS

@@ -1,3 +1,8 @@
+1.5.2
+ * #301: Fix MySQL restore error on "all" database dump by excluding system tables.
+ * Fix PostgreSQL restore error on "all" database dump by using "psql" for the restore instead of
+   "pg_restore".
+
 1.5.1
  * #289: Tired of looking up the latest successful archive name in order to pass it to borgmatic
    actions? Me too. Now you can specify "--archive latest" to all actions that accept an archive

README.md

@@ -130,8 +130,11 @@ Other questions or comments? Contact
 ### Contributing
 
 borgmatic is hosted at <https://torsion.org/borgmatic> with [source code
-available](https://projects.torsion.org/witten/borgmatic). It's also mirrored
-on [GitHub](https://github.com/witten/borgmatic) for convenience.
+available](https://projects.torsion.org/witten/borgmatic), and is also
+mirrored on [GitHub](https://github.com/witten/borgmatic) for convenience.
+
+borgmatic is licensed under the GNU General Public License version 3 or any
+later version.
 
 If you'd like to contribute to borgmatic development, please feel free to
 submit a [Pull Request](https://projects.torsion.org/witten/borgmatic/pulls)

borgmatic/hooks/mysql.py

@@ -16,6 +16,43 @@ def make_dump_path(location_config):  # pragma: no cover
     )
 
 
+SYSTEM_DATABASE_NAMES = ('information_schema', 'mysql', 'performance_schema', 'sys')
+
+
+def database_names_to_dump(database, extra_environment, log_prefix, dry_run_label):
+    '''
+    Given a requested database name, return the corresponding sequence of database names to dump.
+    In the case of "all", query for the names of databases on the configured host and return them,
+    excluding any system databases that will cause problems during restore.
+    '''
+    requested_name = database['name']
+
+    if requested_name != 'all':
+        return (requested_name,)
+
+    show_command = (
+        ('mysql',)
+        + (('--host', database['hostname']) if 'hostname' in database else ())
+        + (('--port', str(database['port'])) if 'port' in database else ())
+        + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+        + (('--user', database['username']) if 'username' in database else ())
+        + ('--skip-column-names', '--batch')
+        + ('--execute', 'show schemas')
+    )
+    logger.debug(
+        '{}: Querying for "all" MySQL databases to dump{}'.format(log_prefix, dry_run_label)
+    )
+    show_output = execute_command(
+        show_command, output_log_level=None, extra_environment=extra_environment
+    )
+
+    return tuple(
+        show_name
+        for show_name in show_output.strip().splitlines()
+        if show_name not in SYSTEM_DATABASE_NAMES
+    )
+
+
 def dump_databases(databases, log_prefix, location_config, dry_run):
     '''
     Dump the given MySQL/MariaDB databases to disk. The databases are supplied as a sequence of
@@ -28,30 +65,37 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
     logger.info('{}: Dumping MySQL databases{}'.format(log_prefix, dry_run_label))
 
     for database in databases:
-        name = database['name']
+        requested_name = database['name']
         dump_filename = dump.make_database_dump_filename(
-            make_dump_path(location_config), name, database.get('hostname')
+            make_dump_path(location_config), requested_name, database.get('hostname')
         )
-        command = (
+        extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
+        dump_command_names = database_names_to_dump(
+            database, extra_environment, log_prefix, dry_run_label
+        )
+
+        dump_command = (
             ('mysqldump', '--add-drop-database')
             + (('--host', database['hostname']) if 'hostname' in database else ())
             + (('--port', str(database['port'])) if 'port' in database else ())
             + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
             + (('--user', database['username']) if 'username' in database else ())
             + (tuple(database['options'].split(' ')) if 'options' in database else ())
-            + (('--all-databases',) if name == 'all' else ('--databases', name))
+            + ('--databases',)
+            + dump_command_names
         )
-        extra_environment = {'MYSQL_PWD': database['password']} if 'password' in database else None
 
         logger.debug(
             '{}: Dumping MySQL database {} to {}{}'.format(
-                log_prefix, name, dump_filename, dry_run_label
+                log_prefix, requested_name, dump_filename, dry_run_label
             )
         )
         if not dry_run:
             os.makedirs(os.path.dirname(dump_filename), mode=0o700, exist_ok=True)
             execute_command(
-                command, output_file=open(dump_filename, 'w'), extra_environment=extra_environment
+                dump_command,
+                output_file=open(dump_filename, 'w'),
+                extra_environment=extra_environment,
             )
 
 

borgmatic/hooks/postgresql.py

@@ -34,7 +34,12 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
         )
         all_databases = bool(name == 'all')
         command = (
-            ('pg_dumpall' if all_databases else 'pg_dump', '--no-password', '--clean')
+            (
+                'pg_dumpall' if all_databases else 'pg_dump',
+                '--no-password',
+                '--clean',
+                '--if-exists',
+            )
             + ('--file', dump_filename)
             + (('--host', database['hostname']) if 'hostname' in database else ())
             + (('--port', str(database['port'])) if 'port' in database else ())
@@ -93,23 +98,28 @@ def restore_database_dumps(databases, log_prefix, location_config, dry_run):
         dump_filename = dump.make_database_dump_filename(
             make_dump_path(location_config), database['name'], database.get('hostname')
         )
-        restore_command = (
-            ('pg_restore', '--no-password', '--clean', '--if-exists', '--exit-on-error')
-            + (('--host', database['hostname']) if 'hostname' in database else ())
-            + (('--port', str(database['port'])) if 'port' in database else ())
-            + (('--username', database['username']) if 'username' in database else ())
-            + ('--dbname', database['name'])
-            + (dump_filename,)
-        )
-        extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
+        all_databases = bool(database['name'] == 'all')
         analyze_command = (
             ('psql', '--no-password', '--quiet')
             + (('--host', database['hostname']) if 'hostname' in database else ())
             + (('--port', str(database['port'])) if 'port' in database else ())
             + (('--username', database['username']) if 'username' in database else ())
-            + ('--dbname', database['name'])
+            + (('--dbname', database['name']) if not all_databases else ())
             + ('--command', 'ANALYZE')
         )
+        restore_command = (
+            ('psql' if all_databases else 'pg_restore', '--no-password')
+            + (
+                ('--if-exists', '--exit-on-error', '--clean', '--dbname', database['name'])
+                if not all_databases
+                else ()
+            )
+            + (('--host', database['hostname']) if 'hostname' in database else ())
+            + (('--port', str(database['port'])) if 'port' in database else ())
+            + (('--username', database['username']) if 'username' in database else ())
+            + (('-f', dump_filename) if all_databases else (dump_filename,))
+        )
+        extra_environment = {'PGPASSWORD': database['password']} if 'password' in database else None
 
         logger.debug(
             '{}: Restoring PostgreSQL database {}{}'.format(

docs/how-to/deal-with-very-large-backups.md

@@ -7,7 +7,7 @@ Borg itself is great for efficiently de-duplicating data across successive
 backup archives, even when dealing with very large repositories. But you may
 find that while borgmatic's default mode of "prune, create, and check" works
 well on small repositories, it's not so great on larger ones. That's because
-running the default consistency checks takes a long time on large
+running the default pruning and consistency checks take a long time on large
 repositories.
 
 ### A la carte actions
@@ -27,9 +27,18 @@ borgmatic check
 
 You can run with only one of these actions provided, or you can mix and match
 any number of them in a single borgmatic run. This supports approaches like
-making backups with `create` on a frequent schedule, while only running
-expensive consistency checks with `check` on a much less frequent basis from
-a separate cron job.
+skipping certain actions while running others. For instance, this skips
+`prune` and only runs `create` and `check`:
+
+```bash
+borgmatic create check
+```
+
+Or, you can make backups with `create` on a frequent schedule (e.g. with
+`borgmatic create` called from one cron job), while only running expensive
+consistency checks with `check` on a much less frequent basis (e.g. with
+`borgmatic check` called from a separate cron job).
+
 
 ### Consistency check configuration
 

docs/how-to/monitor-your-backups.md

@@ -203,12 +203,21 @@ or it doesn't hear from borgmatic for a certain period of time.
 
 ## PagerDuty hook
 
+In case you're new here: [borgmatic](https://torsion.org/borgmatic/) is
+simple, configuration-driven backup software for servers and workstations,
+powered by [Borg Backup](https://www.borgbackup.org/).
+
 [PagerDuty](https://www.pagerduty.com/) provides incident monitoring and
-alerting, and borgmatic has built-in integration with it. Once you create a
-PagerDuty account and <a
+alerting. borgmatic has built-in integration that can notify you via PagerDuty
+as soon as a backup fails, so you can make sure your backups keep working.
+
+First, create a PagerDuty account and <a
 href="https://support.pagerduty.com/docs/services-and-integrations">service</a>
-on their site, all you need to do is configure borgmatic with the unique
-"Integration Key" for your service. Here's an example:
+on their site. On the service, add an integration and set the Integration Type
+to "borgmatic".
+
+Then, configure borgmatic with the unique "Integration Key" for your service.
+Here's an example:
 
 
 ```yaml
@@ -226,6 +235,9 @@ You can configure PagerDuty to notify you by a [variety of
 mechanisms](https://support.pagerduty.com/docs/notifications) when backups
 fail.
 
+If you have any issues with the integration, [please contact
+us](https://torsion.org/borgmatic/#support-and-contributing).
+
 
 ## Scripting borgmatic
 

docs/how-to/set-up-backups.md

@@ -3,6 +3,11 @@ title: How to set up backups with borgmatic
 ---
 ## Installation
 
+Many users need to backup system files that require privileged access, so
+these instructions install and run borgmatic as root. If you don't need to
+backup such files, then you are welcome to install and run borgmatic as a
+non-root user.
+
 First, [install
 Borg](https://borgbackup.readthedocs.io/en/stable/installation.html), at least
 version 1.1.
@@ -39,6 +44,7 @@ borgmatic:
  * [Fedora official](https://bodhi.fedoraproject.org/updates/?search=borgmatic)
  * [Fedora unofficial](https://copr.fedorainfracloud.org/coprs/heffer/borgmatic/)
  * [Arch Linux](https://www.archlinux.org/packages/community/any/borgmatic/)
+ * [Alpine Linux](https://pkgs.alpinelinux.org/packages?name=borgmatic)
  * [OpenBSD](http://ports.su/sysutils/borgmatic)
  * [openSUSE](https://software.opensuse.org/package/borgmatic)
  * [stand-alone binary](https://github.com/cmarquardt/borgmatic-binary)
@@ -132,7 +138,7 @@ this step if you already have a Borg repository.) To create a repository, run
 a command like the following:
 
 ```bash
-borgmatic init --encryption repokey
+sudo borgmatic init --encryption repokey
 ```
 
 (No borgmatic `init` action? Try the old-style `--init` flag, or upgrade
@@ -164,7 +170,7 @@ good idea to test that borgmatic is working. So to run borgmatic and start a
 backup, you can invoke it like this:
 
 ```bash
-borgmatic --verbosity 1
+sudo borgmatic --verbosity 1
 ```
 
 By default, this will also prune any old backups as per the configured

scripts/release

@@ -23,9 +23,10 @@ git push github $version
 rm -fr dist
 python3 setup.py bdist_wheel
 python3 setup.py sdist
-gpg --detach-sign --armor dist/*
-twine upload -r pypi dist/borgmatic-*.tar.gz
-twine upload -r pypi dist/borgmatic-*-py3-none-any.whl
+gpg --detach-sign --armor dist/borgmatic-*.tar.gz
+gpg --detach-sign --armor dist/borgmatic-*-py3-none-any.whl
+twine upload -r pypi dist/borgmatic-*.tar.gz dist/borgmatic-*.tar.gz.asc
+twine upload -r pypi dist/borgmatic-*-py3-none-any.whl borgmatic-*-py3-none-any.whl.asc
 
 # Set release changelogs on projects.torsion.org and GitHub.
 release_changelog="$(cat NEWS | sed '/^$/q' | grep -v '^\S')"

scripts/run-full-dev-tests

@@ -11,4 +11,4 @@
 set -e
 
 docker-compose --file tests/end-to-end/docker-compose.yaml up --force-recreate \
-    --abort-on-container-exit
+    --renew-anon-volumes --abort-on-container-exit

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.5.1'
+VERSION = '1.5.2'
 
 
 setup(

tests/end-to-end/docker-compose.yaml

@@ -1,17 +1,17 @@
 version: '3'
 services:
   postgresql:
-    image: postgres:11.6-alpine
+    image: postgres:12.2-alpine
     environment:
       POSTGRES_PASSWORD: test
       POSTGRES_DB: test
   mysql:
-    image: mariadb:10.4
+    image: mariadb:10.5
     environment:
       MYSQL_ROOT_PASSWORD: test
       MYSQL_DATABASE: test
   tests:
-    image: python:3.7-alpine3.10
+    image: python:3.8-alpine3.11
     volumes:
       - "../..:/app:ro"
     tmpfs:

tests/end-to-end/test_database.py

@@ -29,11 +29,19 @@ hooks:
           hostname: postgresql
           username: postgres
           password: test
+        - name: all
+          hostname: postgresql
+          username: postgres
+          password: test
     mysql_databases:
         - name: test
           hostname: mysql
           username: root
           password: test
+        - name: all
+          hostname: mysql
+          username: root
+          password: test
 '''.format(
         config_path, repository_path, borgmatic_source_directory
     )

tests/unit/hooks/test_mysql.py

@@ -5,6 +5,35 @@ from flexmock import flexmock
 from borgmatic.hooks import mysql as module
 
 
+def test_database_names_to_dump_passes_through_name():
+    extra_environment = flexmock()
+    log_prefix = ''
+    dry_run_label = ''
+
+    names = module.database_names_to_dump(
+        {'name': 'foo'}, extra_environment, log_prefix, dry_run_label
+    )
+
+    assert names == ('foo',)
+
+
+def test_database_names_to_dump_queries_mysql_for_database_names():
+    extra_environment = flexmock()
+    log_prefix = ''
+    dry_run_label = ''
+    flexmock(module).should_receive('execute_command').with_args(
+        ('mysql', '--skip-column-names', '--batch', '--execute', 'show schemas'),
+        output_log_level=None,
+        extra_environment=extra_environment,
+    ).and_return('foo\nbar\nmysql\n').once()
+
+    names = module.database_names_to_dump(
+        {'name': 'all'}, extra_environment, log_prefix, dry_run_label
+    )
+
+    assert names == ('foo', 'bar')
+
+
 def test_dump_databases_runs_mysqldump_for_each_database():
     databases = [{'name': 'foo'}, {'name': 'bar'}]
     output_file = flexmock()
@@ -12,6 +41,9 @@ def test_dump_databases_runs_mysqldump_for_each_database():
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     ).and_return('databases/localhost/bar')
+    flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
+        ('bar',)
+    )
     flexmock(module.os).should_receive('makedirs')
     flexmock(sys.modules['builtins']).should_receive('open').and_return(output_file)
 
@@ -31,6 +63,9 @@ def test_dump_databases_with_dry_run_skips_mysqldump():
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     ).and_return('databases/localhost/bar')
+    flexmock(module).should_receive('database_names_to_dump').and_return(('foo',)).and_return(
+        ('bar',)
+    )
     flexmock(module.os).should_receive('makedirs').never()
     flexmock(module).should_receive('execute_command').never()
 
@@ -44,6 +79,7 @@ def test_dump_databases_runs_mysqldump_with_hostname_and_port():
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/database.example.org/foo'
     )
+    flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
     flexmock(module.os).should_receive('makedirs')
     flexmock(sys.modules['builtins']).should_receive('open').and_return(output_file)
 
@@ -74,6 +110,7 @@ def test_dump_databases_runs_mysqldump_with_username_and_password():
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     )
+    flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
     flexmock(module.os).should_receive('makedirs')
     flexmock(sys.modules['builtins']).should_receive('open').and_return(output_file)
 
@@ -93,6 +130,7 @@ def test_dump_databases_runs_mysqldump_with_options():
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/foo'
     )
+    flexmock(module).should_receive('database_names_to_dump').and_return(('foo',))
     flexmock(module.os).should_receive('makedirs')
     flexmock(sys.modules['builtins']).should_receive('open').and_return(output_file)
 
@@ -112,11 +150,12 @@ def test_dump_databases_runs_mysqldump_for_all_databases():
     flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
         'databases/localhost/all'
     )
+    flexmock(module).should_receive('database_names_to_dump').and_return(('foo', 'bar'))
     flexmock(module.os).should_receive('makedirs')
     flexmock(sys.modules['builtins']).should_receive('open').and_return(output_file)
 
     flexmock(module).should_receive('execute_command').with_args(
-        ('mysqldump', '--add-drop-database', '--all-databases'),
+        ('mysqldump', '--add-drop-database', '--databases', 'foo', 'bar'),
         output_file=output_file,
         extra_environment=None,
     ).once()

tests/unit/hooks/test_postgresql.py

@@ -17,6 +17,7 @@ def test_dump_databases_runs_pg_dump_for_each_database():
                 'pg_dump',
                 '--no-password',
                 '--clean',
+                '--if-exists',
                 '--file',
                 'databases/localhost/{}'.format(name),
                 '--format',
@@ -54,6 +55,7 @@ def test_dump_databases_runs_pg_dump_with_hostname_and_port():
             'pg_dump',
             '--no-password',
             '--clean',
+            '--if-exists',
             '--file',
             'databases/database.example.org/foo',
             '--host',
@@ -83,6 +85,7 @@ def test_dump_databases_runs_pg_dump_with_username_and_password():
             'pg_dump',
             '--no-password',
             '--clean',
+            '--if-exists',
             '--file',
             'databases/localhost/foo',
             '--username',
@@ -110,6 +113,7 @@ def test_dump_databases_runs_pg_dump_with_format():
             'pg_dump',
             '--no-password',
             '--clean',
+            '--if-exists',
             '--file',
             'databases/localhost/foo',
             '--format',
@@ -135,6 +139,7 @@ def test_dump_databases_runs_pg_dump_with_options():
             'pg_dump',
             '--no-password',
             '--clean',
+            '--if-exists',
             '--file',
             'databases/localhost/foo',
             '--format',
@@ -157,7 +162,14 @@ def test_dump_databases_runs_pg_dumpall_for_all_databases():
     flexmock(module.os).should_receive('makedirs')
 
     flexmock(module).should_receive('execute_command').with_args(
-        ('pg_dumpall', '--no-password', '--clean', '--file', 'databases/localhost/all'),
+        (
+            'pg_dumpall',
+            '--no-password',
+            '--clean',
+            '--if-exists',
+            '--file',
+            'databases/localhost/all',
+        ),
         extra_environment=None,
     ).once()
 
@@ -197,9 +209,9 @@ def test_restore_database_dumps_restores_each_database():
             (
                 'pg_restore',
                 '--no-password',
-                '--clean',
                 '--if-exists',
                 '--exit-on-error',
+                '--clean',
                 '--dbname',
                 name,
                 'databases/localhost/{}'.format(name),
@@ -225,15 +237,15 @@ def test_restore_database_dumps_runs_pg_restore_with_hostname_and_port():
         (
             'pg_restore',
             '--no-password',
-            '--clean',
             '--if-exists',
             '--exit-on-error',
+            '--clean',
+            '--dbname',
+            'foo',
             '--host',
             'database.example.org',
             '--port',
             '5433',
-            '--dbname',
-            'foo',
             'databases/localhost/foo',
         ),
         extra_environment=None,
@@ -269,13 +281,13 @@ def test_restore_database_dumps_runs_pg_restore_with_username_and_password():
         (
             'pg_restore',
             '--no-password',
-            '--clean',
             '--if-exists',
             '--exit-on-error',
-            '--username',
-            'postgres',
+            '--clean',
             '--dbname',
             'foo',
+            '--username',
+            'postgres',
             'databases/localhost/foo',
         ),
         extra_environment={'PGPASSWORD': 'trustsome1'},
@@ -296,3 +308,20 @@ def test_restore_database_dumps_runs_pg_restore_with_username_and_password():
     ).once()
 
     module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)
+
+
+def test_restore_database_dumps_runs_psql_for_all_database_dump():
+    databases = [{'name': 'all'}]
+    flexmock(module).should_receive('make_dump_path').and_return('')
+    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
+        'databases/localhost/all'
+    )
+
+    flexmock(module).should_receive('execute_command').with_args(
+        ('psql', '--no-password', '-f', 'databases/localhost/all'), extra_environment=None
+    ).once()
+    flexmock(module).should_receive('execute_command').with_args(
+        ('psql', '--no-password', '--quiet', '--command', 'ANALYZE'), extra_environment=None
+    ).once()
+
+    module.restore_database_dumps(databases, 'test.yaml', {}, dry_run=False)