Bump version for release.

Reviewed-on: https://projects.torsion.org/borgmatic-collective/borgmatic/pulls/464

.drone.yml

@@ -82,9 +82,7 @@ clone:
 
 steps:
 - name: build
-  #image: plugins/docker
-  # Temporary work-around for https://github.com/drone-plugins/drone-docker/pull/327
-  image: techknowlogick/drone-docker
+  image: plugins/docker
   settings:
     username:
       from_secret: docker_username
@@ -94,5 +92,9 @@ steps:
     dockerfile: docs/Dockerfile
 
 trigger:
+  repo:
+    - borgmatic-collective/borgmatic
   branch:
     - master
+  event:
+    - push

NEWS

@@ -1,4 +1,13 @@
-1.5.19.dev0
+1.5.21
+ * #28: Optionally retry failing backups via "retries" and "retry_wait" configuration options.
+ * #306: Add "list_options" MySQL configuration option for passing additional arguments to MySQL
+   list command.
+ * #459: Add support for old version (2.x) of jsonschema library.
+
+1.5.20
+ * Re-release with correct version without dev0 tag.
+
+1.5.19
  * #387: Fix error when configured source directories are not present on the filesystem at the time
    of backup. Now, Borg will complain, but the backup will still continue.
  * #455: Mention changing borgmatic path in cron documentation.

README.md

@@ -92,14 +92,13 @@ development and hosting when you use these links to sign up. (These are
 referral links, but without any tracking scripts or cookies.)
 
 <ul>
- <li class="referral"><a href="https://www.rsync.net/cgi-bin/borg.cgi?campaign=borg&adgroup=borgmatic">rsync.net</a>: Cloud Storage provider with full support for borg and any other SSH/SFTP tool</li>
  <li class="referral"><a href="https://www.borgbase.com/?utm_source=borgmatic">BorgBase</a>: Borg hosting service with support for monitoring, 2FA, and append-only repos</li>
  <li class="referral"><a href="https://storage.lima-labs.com/special-pricing-offer-for-borgmatic-users/">Lima-Labs</a>: Affordable, reliable cloud data storage accessable via SSH/SCP/FTP for Borg backups or any other bulk storage needs</li>
 </ul>
 
-Additionally, [Hetzner](https://www.hetzner.com/storage/storage-box) has a
-compatible storage offering, but does not currently fund borgmatic
-development or hosting.
+Additionally, [rsync.net](https://www.rsync.net/products/borg.html) and
+[Hetzner](https://www.hetzner.com/storage/storage-box) have compatible storage
+offerings, but do not currently fund borgmatic development or hosting.
 
 ## Support and contributing
 

borgmatic/commands/borgmatic.py

@@ -4,6 +4,8 @@ import json
 import logging
 import os
 import sys
+import time
+from queue import Queue
 from subprocess import CalledProcessError
 
 import colorama
@@ -52,6 +54,8 @@ def run_configuration(config_filename, config, arguments):
 
     local_path = location.get('local_path', 'borg')
     remote_path = location.get('remote_path')
+    retries = storage.get('retries', 0)
+    retry_wait = storage.get('retry_wait', 0)
     borg_environment.initialize(storage)
     encountered_error = None
     error_repository = ''
@@ -120,7 +124,16 @@ def run_configuration(config_filename, config, arguments):
         )
 
     if not encountered_error:
-        for repository_path in location['repositories']:
+        repo_queue = Queue()
+        for repo in location['repositories']:
+            repo_queue.put((repo, 0),)
+
+        while not repo_queue.empty():
+            repository_path, retry_num = repo_queue.get()
+            timeout = retry_num * retry_wait
+            if timeout:
+                logger.warning(f'{config_filename}: Sleeping {timeout}s before next retry')
+                time.sleep(timeout)
             try:
                 yield from run_actions(
                     arguments=arguments,
@@ -134,11 +147,17 @@ def run_configuration(config_filename, config, arguments):
                     repository_path=repository_path,
                 )
             except (OSError, CalledProcessError, ValueError) as error:
-                encountered_error = error
-                error_repository = repository_path
                 yield from make_error_log_records(
                     '{}: Error running actions for repository'.format(repository_path), error
                 )
+                if retry_num < retries:
+                    repo_queue.put((repository_path, retry_num + 1),)
+                    logger.warning(
+                        f'{config_filename}: Retrying... attempt {retry_num + 1}/{retries}'
+                    )
+                    continue
+                encountered_error = error
+                error_repository = repository_path
 
     if not encountered_error:
         try:
@@ -257,7 +276,7 @@ def run_actions(
     hooks,
     local_path,
     remote_path,
-    repository_path
+    repository_path,
 ):  # pragma: no cover
     '''
     Given parsed command-line arguments as an argparse.ArgumentParser instance, several different

borgmatic/config/schema.yaml

@@ -251,6 +251,19 @@ properties:
                     Remote network upload rate limit in kiBytes/second. Defaults
                     to unlimited.
                 example: 100
+            retries:
+                type: integer
+                description: |
+                    Number of times to retry a failing backup before giving up.
+                    Defaults to 0 (i.e., does not attempt retry).
+                example: 3
+            retry_wait:
+                type: integer
+                description: |
+                    Wait time between retries (in seconds) to allow transient
+                    issues to pass. Increases after each retry as a form of
+                    backoff. Defaults to 0 (no wait).
+                example: 10
             temporary_directory:
                 type: string
                 description: |
@@ -736,6 +749,14 @@ properties:
                                 configured to trust the configured username
                                 without a password.
                             example: trustsome1
+                        list_options:
+                            type: string
+                            description: |
+                                Additional mysql options to pass directly to
+                                the mysql command that lists available
+                                databases, without performing any validation on
+                                them. See mysql documentation for details.
+                            example: --defaults-extra-file=my.cnf
                         options:
                             type: string
                             description: |

borgmatic/config/validate.py

@@ -110,7 +110,10 @@ def parse_configuration(config_filename, schema_filename, overrides=None):
     override.apply_overrides(config, overrides)
     normalize.normalize(config)
 
-    validator = jsonschema.Draft7Validator(schema)
+    try:
+        validator = jsonschema.Draft7Validator(schema)
+    except AttributeError:  # pragma: no cover
+        validator = jsonschema.Draft4Validator(schema)
     validation_errors = tuple(validator.iter_errors(config))
 
     if validation_errors:

borgmatic/hooks/mysql.py

@@ -31,6 +31,7 @@ def database_names_to_dump(database, extra_environment, log_prefix, dry_run_labe
 
     show_command = (
         ('mysql',)
+        + (tuple(database['list_options'].split(' ')) if 'list_options' in database else ())
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())

docs/how-to/develop-on-borgmatic.md

@@ -20,7 +20,7 @@ git clone ssh://git@projects.torsion.org:3022/borgmatic-collective/borgmatic.git
 ```
 
 Then, install borgmatic
-"[editable](https://pip.pypa.io/en/stable/reference/pip_install/#editable-installs)"
+"[editable](https://pip.pypa.io/en/stable/cli/pip_install/#editable-installs)"
 so that you can run borgmatic commands while you're hacking on them to
 make sure your changes work.
 

docs/how-to/set-up-backups.md

@@ -100,14 +100,13 @@ development and hosting when you use these links to sign up. (These are
 referral links, but without any tracking scripts or cookies.)
 
 <ul>
- <li class="referral"><a href="https://www.rsync.net/cgi-bin/borg.cgi?campaign=borg&adgroup=borgmatic">rsync.net</a>: Cloud Storage provider with full support for borg and any other SSH/SFTP tool</li>
  <li class="referral"><a href="https://www.borgbase.com/?utm_source=borgmatic">BorgBase</a>: Borg hosting service with support for monitoring, 2FA, and append-only repos</li>
  <li class="referral"><a href="https://storage.lima-labs.com/special-pricing-offer-for-borgmatic-users/">Lima-Labs</a>: Affordable, reliable cloud data storage accessable via SSH/SCP/FTP for Borg backups or any other bulk storage needs</li>
 </ul>
 
-Additionally, [Hetzner](https://www.hetzner.com/storage/storage-box) has a
-compatible storage offering, but does not currently fund borgmatic
-development or hosting.
+Additionally, [rsync.net](https://www.rsync.net/products/borg.html) and
+[Hetzner](https://www.hetzner.com/storage/storage-box) have compatible storage
+offerings, but do not currently fund borgmatic development or hosting.
 
 ## Configuration
 

scripts/run-full-tests

@@ -13,8 +13,8 @@ set -e
 apk add --no-cache python3 py3-pip borgbackup postgresql-client mariadb-client
 # If certain dependencies of black are available in this version of Alpine, install them.
 apk add --no-cache py3-typed-ast py3-regex || true
-python3 -m pip install --upgrade pip==20.2.4 setuptools==50.3.2
-pip3 install tox==3.20.1
+python3 -m pip install --upgrade pip==21.3.1 setuptools==58.2.0
+pip3 install tox==3.24.4
 export COVERAGE_FILE=/tmp/.coverage
 tox --workdir /tmp/.tox --sitepackages
 tox --workdir /tmp/.tox --sitepackages -e end-to-end

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.5.19.dev0'
+VERSION = '1.5.21'
 
 
 setup(

tests/unit/commands/test_borgmatic.py

@@ -1,5 +1,6 @@
 import logging
 import subprocess
+import time
 
 from flexmock import flexmock
 
@@ -184,6 +185,160 @@ def test_run_configuration_bails_for_on_error_hook_soft_failure():
     assert results == expected_results
 
 
+def test_run_configuration_retries_soft_error():
+    # Run action first fails, second passes
+    flexmock(module.borg_environment).should_receive('initialize')
+    flexmock(module.command).should_receive('execute_hook')
+    flexmock(module).should_receive('run_actions').and_raise(OSError).and_return([])
+    expected_results = [flexmock()]
+    flexmock(module).should_receive('make_error_log_records').and_return(expected_results).once()
+    config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 1}}
+    arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
+    results = list(module.run_configuration('test.yaml', config, arguments))
+    assert results == expected_results
+
+
+def test_run_configuration_retries_hard_error():
+    # Run action fails twice
+    flexmock(module.borg_environment).should_receive('initialize')
+    flexmock(module.command).should_receive('execute_hook')
+    flexmock(module).should_receive('run_actions').and_raise(OSError).times(2)
+    expected_results = [flexmock(), flexmock()]
+    flexmock(module).should_receive('make_error_log_records').with_args(
+        'foo: Error running actions for repository', OSError
+    ).and_return(expected_results[:1]).with_args(
+        'foo: Error running actions for repository', OSError
+    ).and_return(
+        expected_results[1:]
+    ).twice()
+    config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 1}}
+    arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
+    results = list(module.run_configuration('test.yaml', config, arguments))
+    assert results == expected_results
+
+
+def test_run_repos_ordered():
+    flexmock(module.borg_environment).should_receive('initialize')
+    flexmock(module.command).should_receive('execute_hook')
+    flexmock(module).should_receive('run_actions').and_raise(OSError).times(2)
+    expected_results = [flexmock(), flexmock()]
+    flexmock(module).should_receive('make_error_log_records').with_args(
+        'foo: Error running actions for repository', OSError
+    ).and_return(expected_results[:1]).ordered()
+    flexmock(module).should_receive('make_error_log_records').with_args(
+        'bar: Error running actions for repository', OSError
+    ).and_return(expected_results[1:]).ordered()
+    config = {'location': {'repositories': ['foo', 'bar']}}
+    arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
+    results = list(module.run_configuration('test.yaml', config, arguments))
+    assert results == expected_results
+
+
+def test_run_configuration_retries_round_robbin():
+    flexmock(module.borg_environment).should_receive('initialize')
+    flexmock(module.command).should_receive('execute_hook')
+    flexmock(module).should_receive('run_actions').and_raise(OSError).times(4)
+    expected_results = [flexmock(), flexmock(), flexmock(), flexmock()]
+    flexmock(module).should_receive('make_error_log_records').with_args(
+        'foo: Error running actions for repository', OSError
+    ).and_return(expected_results[0:1]).ordered()
+    flexmock(module).should_receive('make_error_log_records').with_args(
+        'bar: Error running actions for repository', OSError
+    ).and_return(expected_results[1:2]).ordered()
+    flexmock(module).should_receive('make_error_log_records').with_args(
+        'foo: Error running actions for repository', OSError
+    ).and_return(expected_results[2:3]).ordered()
+    flexmock(module).should_receive('make_error_log_records').with_args(
+        'bar: Error running actions for repository', OSError
+    ).and_return(expected_results[3:4]).ordered()
+    config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}}
+    arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
+    results = list(module.run_configuration('test.yaml', config, arguments))
+    assert results == expected_results
+
+
+def test_run_configuration_retries_one_passes():
+    flexmock(module.borg_environment).should_receive('initialize')
+    flexmock(module.command).should_receive('execute_hook')
+    flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return(
+        []
+    ).and_raise(OSError).times(4)
+    expected_results = [flexmock(), flexmock(), flexmock()]
+    flexmock(module).should_receive('make_error_log_records').with_args(
+        'foo: Error running actions for repository', OSError
+    ).and_return(expected_results[0:1]).ordered()
+    flexmock(module).should_receive('make_error_log_records').with_args(
+        'bar: Error running actions for repository', OSError
+    ).and_return(expected_results[1:2]).ordered()
+    flexmock(module).should_receive('make_error_log_records').with_args(
+        'bar: Error running actions for repository', OSError
+    ).and_return(expected_results[2:3]).ordered()
+    config = {'location': {'repositories': ['foo', 'bar']}, 'storage': {'retries': 1}}
+    arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
+    results = list(module.run_configuration('test.yaml', config, arguments))
+    assert results == expected_results
+
+
+def test_run_configuration_retry_wait():
+    flexmock(module.borg_environment).should_receive('initialize')
+    flexmock(module.command).should_receive('execute_hook')
+    flexmock(module).should_receive('run_actions').and_raise(OSError).times(4)
+    expected_results = [flexmock(), flexmock(), flexmock(), flexmock()]
+    flexmock(module).should_receive('make_error_log_records').with_args(
+        'foo: Error running actions for repository', OSError
+    ).and_return(expected_results[0:1]).ordered()
+
+    flexmock(time).should_receive('sleep').with_args(10).and_return().ordered()
+    flexmock(module).should_receive('make_error_log_records').with_args(
+        'foo: Error running actions for repository', OSError
+    ).and_return(expected_results[1:2]).ordered()
+
+    flexmock(time).should_receive('sleep').with_args(20).and_return().ordered()
+    flexmock(module).should_receive('make_error_log_records').with_args(
+        'foo: Error running actions for repository', OSError
+    ).and_return(expected_results[2:3]).ordered()
+
+    flexmock(time).should_receive('sleep').with_args(30).and_return().ordered()
+    flexmock(module).should_receive('make_error_log_records').with_args(
+        'foo: Error running actions for repository', OSError
+    ).and_return(expected_results[3:4]).ordered()
+    config = {'location': {'repositories': ['foo']}, 'storage': {'retries': 3, 'retry_wait': 10}}
+    arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
+    results = list(module.run_configuration('test.yaml', config, arguments))
+    assert results == expected_results
+
+
+def test_run_configuration_retries_timeout_multiple_repos():
+    flexmock(module.borg_environment).should_receive('initialize')
+    flexmock(module.command).should_receive('execute_hook')
+    flexmock(module).should_receive('run_actions').and_raise(OSError).and_raise(OSError).and_return(
+        []
+    ).and_raise(OSError).times(4)
+    expected_results = [flexmock(), flexmock(), flexmock()]
+    flexmock(module).should_receive('make_error_log_records').with_args(
+        'foo: Error running actions for repository', OSError
+    ).and_return(expected_results[0:1]).ordered()
+    flexmock(module).should_receive('make_error_log_records').with_args(
+        'bar: Error running actions for repository', OSError
+    ).and_return(expected_results[1:2]).ordered()
+
+    # Sleep before retrying foo (and passing)
+    flexmock(time).should_receive('sleep').with_args(10).and_return().ordered()
+
+    # Sleep before retrying bar (and failing)
+    flexmock(time).should_receive('sleep').with_args(10).and_return().ordered()
+    flexmock(module).should_receive('make_error_log_records').with_args(
+        'bar: Error running actions for repository', OSError
+    ).and_return(expected_results[2:3]).ordered()
+    config = {
+        'location': {'repositories': ['foo', 'bar']},
+        'storage': {'retries': 1, 'retry_wait': 10},
+    }
+    arguments = {'global': flexmock(monitoring_verbosity=1, dry_run=False), 'create': flexmock()}
+    results = list(module.run_configuration('test.yaml', config, arguments))
+    assert results == expected_results
+
+
 def test_load_configurations_collects_parsed_configurations():
     configuration = flexmock()
     other_configuration = flexmock()

tests/unit/hooks/test_mysql.py

@@ -198,6 +198,24 @@ def test_dump_databases_runs_mysqldump_for_all_databases():
     assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
 
 
+def test_database_names_to_dump_runs_mysql_with_list_options():
+    database = {'name': 'all', 'list_options': '--defaults-extra-file=my.cnf'}
+    flexmock(module).should_receive('execute_command').with_args(
+        (
+            'mysql',
+            '--defaults-extra-file=my.cnf',
+            '--skip-column-names',
+            '--batch',
+            '--execute',
+            'show schemas',
+        ),
+        output_log_level=None,
+        extra_environment=None,
+    ).and_return(('foo\nbar')).once()
+
+    assert module.database_names_to_dump(database, None, 'test.yaml', '') == ('foo', 'bar')
+
+
 def test_dump_databases_errors_for_missing_all_databases():
     databases = [{'name': 'all'}]
     process = flexmock()