Rev to 1.2.8.

.drone.yml

@@ -1,9 +1,17 @@
 pipeline:
   build:
-    image: python:3.7.0-alpine3.8
+    image: python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
     pull: true
     commands:
       - pip install tox
       - tox
       - apk add --no-cache borgbackup
       - tox -e end-to-end
+
+matrix:
+  ALPINE_VERSION:
+    - 3.7
+    - 3.8
+  PYTHON_VERSION:
+    - 3.6
+    - 3.7

NEWS

@@ -1,3 +1,9 @@
+1.2.8
+ * #73: Enable consistency checks for only certain repositories via "check_repositories" option in
+   borgmatic's consistency configuration. Handy for large repositories that take forever to check.
+ * Include link to issue tracker within various command output.
+ * Run continuous integration tests on a matrix of Python and Borg versions.
+
 1.2.7
  * #98: Support for Borg --keep-secondly prune option.
  * Use Black code formatter and Flake8 code checker as part of running automated tests.

README.md

@@ -56,13 +56,17 @@ href="https://asciinema.org/a/203761" target="_blank">screencast</a>.
 
 ## Installation
 
-To get up and running, follow the [Borg Quick
+To get up and running, first [install
+Borg](https://borgbackup.readthedocs.io/en/latest/installation.html), at
+least version 1.1. Then, follow the [Borg Quick
 Start](https://borgbackup.readthedocs.org/en/latest/quickstart.html) to create
-a repository on a local or remote host. Note that if you plan to run borgmatic
+a repository on a local or remote host.
-on a schedule with cron, and you encrypt your Borg repository with a
+
-passphrase instead of a key file, you'll either need to set the borgmatic
+Note that if you plan to run borgmatic on a schedule with cron, and you
-`encryption_passphrase` configuration variable or set the `BORG_PASSPHRASE`
+encrypt your Borg repository with a passphrase instead of a key file, you'll
-environment variable. See the [repository encryption
+either need to set the borgmatic `encryption_passphrase` configuration
+variable or set the `BORG_PASSPHRASE` environment variable. See the
+[repository encryption
 section](https://borgbackup.readthedocs.io/en/latest/quickstart.html#repository-encryption)
 of the Quick Start for more info.
 
@@ -478,7 +482,7 @@ consistency:
 ```
 
 This error can be caused by an ssh timeout, which you can rectify by adding
-the following to the ~/.ssh/config file on the client:
+the following to the `~/.ssh/config` file on the client:
 
 ```text
     Host *

borgmatic/commands/borgmatic.py

@@ -13,7 +13,7 @@ from borgmatic.borg import (
     info as borg_info,
 )
 from borgmatic.commands import hook
-from borgmatic.config import collect, convert, validate
+from borgmatic.config import checks, collect, convert, validate
 from borgmatic.signals import configure_signals
 from borgmatic.verbosity import verbosity_to_log_level
 
@@ -144,7 +144,15 @@ def run_configuration(config_filename, args):  # pragma: no cover
         if args.create:
             hook.execute_hook(hooks.get('before_backup'), config_filename, 'pre-backup')
 
-        _run_commands(args, consistency, local_path, location, remote_path, retention, storage)
+        _run_commands(
+            args=args,
+            consistency=consistency,
+            local_path=local_path,
+            location=location,
+            remote_path=remote_path,
+            retention=retention,
+            storage=storage,
+        )
 
         if args.create:
             hook.execute_hook(hooks.get('after_backup'), config_filename, 'post-backup')
@@ -153,25 +161,26 @@ def run_configuration(config_filename, args):  # pragma: no cover
         raise
 
 
-def _run_commands(args, consistency, local_path, location, remote_path, retention, storage):
+def _run_commands(*, args, consistency, local_path, location, remote_path, retention, storage):
     json_results = []
     for unexpanded_repository in location['repositories']:
         _run_commands_on_repository(
-            args,
+            args=args,
-            consistency,
+            consistency=consistency,
-            json_results,
+            json_results=json_results,
-            local_path,
+            local_path=local_path,
-            location,
+            location=location,
-            remote_path,
+            remote_path=remote_path,
-            retention,
+            retention=retention,
-            storage,
+            storage=storage,
-            unexpanded_repository,
+            unexpanded_repository=unexpanded_repository,
         )
     if args.json:
         sys.stdout.write(json.dumps(json_results))
 
 
 def _run_commands_on_repository(
+    *,
     args,
     consistency,
     json_results,
@@ -204,7 +213,7 @@ def _run_commands_on_repository(
             local_path=local_path,
             remote_path=remote_path,
         )
-    if args.check:
+    if args.check and checks.repository_enabled_for_checks(repository, consistency):
         logger.info('{}: Running consistency checks'.format(repository))
         borg_check.check_archives(
             repository, storage, consistency, local_path=local_path, remote_path=remote_path
@@ -248,4 +257,6 @@ def main():  # pragma: no cover
             run_configuration(config_filename, args)
     except (ValueError, OSError, CalledProcessError) as error:
         print(error, file=sys.stderr)
+        print(file=sys.stderr)
+        print('Need some help? https://torsion.org/borgmatic/#issues', file=sys.stderr)
         sys.exit(1)

borgmatic/commands/generate_config.py

@@ -38,6 +38,8 @@ def main():  # pragma: no cover
         print()
         print('Please edit the file to suit your needs. The values are just representative.')
         print('All fields are optional except where indicated.')
+        print()
+        print('If you ever need help: https://torsion.org/borgmatic/#issues')
     except (ValueError, OSError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)

borgmatic/config/checks.py

@@ -0,0 +1,9 @@
+def repository_enabled_for_checks(repository, consistency):
+    '''
+    Given a repository name and a consistency configuration dict, return whether the repository
+    is enabled to have consistency checks run.
+    '''
+    if not consistency.get('check_repositories'):
+        return True
+
+    return repository in consistency['check_repositories']

borgmatic/config/schema.yaml

@@ -236,6 +236,16 @@ map:
                 example:
                     - repository
                     - archives
+            check_repositories:
+                seq:
+                    - type: scalar
+                desc: |
+                    Paths to a subset of the repositories in the location section on which to run
+                    consistency checks. Handy in case some of your repositories are very large, and
+                    so running consistency checks on them would take too long. Defaults to running
+                    consistency checks on all repositories configured in the location section.
+                example:
+                    - user@backupserver:sourcehostname.borg
             check_last:
                 type: int
                 desc: Restrict the number of checked archives to the last n. Applies only to the

borgmatic/config/validate.py

@@ -51,6 +51,19 @@ def apply_logical_validation(config_filename, parsed_configuration):
             ('If you provide an archive_name_format, you must also specify a retention prefix.',),
         )
 
+    location_repositories = parsed_configuration.get('location', {}).get('repositories')
+    check_repositories = parsed_configuration.get('consistency', {}).get('check_repositories', [])
+    for repository in check_repositories:
+        if repository not in location_repositories:
+            raise Validation_error(
+                config_filename,
+                (
+                    'Unknown repository in the consistency section\'s check_repositories: {}'.format(
+                        repository
+                    ),
+                ),
+            )
+
     consistency_prefix = parsed_configuration.get('consistency', {}).get('prefix')
     if archive_name_format and not consistency_prefix:
         logger.warning(

scripts/release

@@ -2,13 +2,38 @@
 
 set -e
 
+projects_token=${1:-}
+github_token=${2:-}
+
+if [[ -z $github_token ]]; then
+    echo "Usage: $0 [projects-token] [github-token]"
+    exit 1
+fi
+if [[ ! -f NEWS ]]; then
+    echo "Missing NEWS file. Try running from root of repository."
+    exit 1
+fi
+
 version=$(head --lines=1 NEWS)
 git tag $version
 git push origin $version
 git push github $version
 
+# Build borgmatic and publish to pypi.
 rm -fr dist
 python3 setup.py bdist_wheel
 python3 setup.py sdist
 twine upload -r pypi dist/borgmatic-*.tar.gz
 twine upload -r pypi dist/borgmatic-*-py3-none-any.whl
+
+# Set release changelogs on projects.evoworx.org and GitHub.
+release_changelog=$(sed '/^$/q' NEWS |grep '^\s*\*')
+escaped_release_changelog=$(echo $release_changelog | sed -z 's/\n/\\n/g')
+curl --request POST \
+    "https://projects.torsion.org/api/v1/repos/witten/borgmatic/releases?access_token=$projects_token" \
+    --header "Accept: application/json" \
+    --header "Content-Type: application/json" \
+    --data "{\"body\": \"$escaped_release_changelog\", \"draft\": false, \"name\": \"borgmatic $version\", \"prerelease\": false, \"tag_name\": \"$version\"}"
+
+github-release create --token="$github_token" --owner=witten --repo=borgmatic --tag="$version" \
+    --name="borgmatic $version" --body="$release_changelog"

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.2.7'
+VERSION = '1.2.8'
 
 
 setup(

test_requirements.txt

@@ -1,7 +1,23 @@
+appdirs==1.4.3
+atomicwrites==1.2.1
+attrs==18.2.0
 black==18.9b0
+Click==7.0
+coverage==4.5.1
+docopt==0.6.2
 flake8==3.5.0
 flexmock==0.10.2
+mccabe==0.6.1
+more-itertools==4.3.0
+pluggy==0.7.1
+py==1.6.0
+pycodestyle==2.3.1
+pyflakes==2.0.0
 pykwalify==1.7.0
-pytest==3.8.1
+pytest==3.8.2
 pytest-cov==2.6.0
+python-dateutil==2.7.3
+PyYAML==3.13
 ruamel.yaml>0.15.0,<0.16.0
+six==1.11.0
+toml==0.10.0

tests/end-to-end/test_borgmatic.py

@@ -49,5 +49,15 @@ def test_borgmatic_command():
 
         assert len(parsed_output) == 1
         assert len(parsed_output[0]['archives']) == 1
+
+        # Also exercise the info flag.
+        output = subprocess.check_output(
+            f'borgmatic --config {config_path} --info --json'.split(' '),
+            encoding=sys.stdout.encoding,
+        )
+        parsed_output = json.loads(output)
+
+        assert len(parsed_output) == 1
+        assert 'repository' in parsed_output[0]
     finally:
         shutil.rmtree(temporary_directory)

tests/unit/commands/test_borgmatic.py

@@ -6,7 +6,7 @@ from flexmock import flexmock
 from borgmatic.commands import borgmatic
 
 
-def test__run_commands_handles_multiple_json_outputs_in_array():
+def test_run_commands_handles_multiple_json_outputs_in_array():
     (
         flexmock(borgmatic)
         .should_receive('_run_commands_on_repository')

tests/unit/config/test_checks.py

@@ -0,0 +1,23 @@
+from borgmatic.config import checks as module
+
+
+def test_repository_enabled_for_checks_defaults_to_enabled_for_all_repositories():
+    enabled = module.repository_enabled_for_checks('repo.borg', consistency={})
+
+    assert enabled
+
+
+def test_repository_enabled_for_checks_is_enabled_for_specified_repositories():
+    enabled = module.repository_enabled_for_checks(
+        'repo.borg', consistency={'check_repositories': ['repo.borg', 'other.borg']}
+    )
+
+    assert enabled
+
+
+def test_repository_enabled_for_checks_is_disabled_for_other_repositories():
+    enabled = module.repository_enabled_for_checks(
+        'repo.borg', consistency={'check_repositories': ['other.borg']}
+    )
+
+    assert not enabled

tests/unit/config/test_validate.py

@@ -51,6 +51,29 @@ def test_apply_logical_validation_warns_if_archive_name_format_present_without_c
     )
 
 
+def test_apply_locical_validation_raises_if_unknown_repository_in_check_repositories():
+    with pytest.raises(module.Validation_error):
+        module.apply_logical_validation(
+            'config.yaml',
+            {
+                'location': {'repositories': ['repo.borg', 'other.borg']},
+                'retention': {'keep_secondly': 1000},
+                'consistency': {'check_repositories': ['repo.borg', 'unknown.borg']},
+            },
+        )
+
+
+def test_apply_locical_validation_does_not_raise_if_known_repository_in_check_repositories():
+    module.apply_logical_validation(
+        'config.yaml',
+        {
+            'location': {'repositories': ['repo.borg', 'other.borg']},
+            'retention': {'keep_secondly': 1000},
+            'consistency': {'check_repositories': ['repo.borg']},
+        },
+    )
+
+
 def test_apply_logical_validation_does_not_raise_or_warn_if_archive_name_format_and_prefix_present():
     logger = flexmock(module.logger)
     logger.should_receive('warning').never()