Support for Borg --chunker-params create option via "chunker_params" in borgmatic's storage section (#105).

.drone.yml

@@ -1,9 +1,18 @@
 pipeline:
   build:
-    image: python:3.7.0-alpine3.8
+    image: python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
     pull: true
     commands:
       - pip install tox
       - tox
       - apk add --no-cache borgbackup
       - tox -e end-to-end
+
+matrix:
+  ALPINE_VERSION:
+    - 3.7
+    - 3.8
+  PYTHON_VERSION:
+    - 3.5
+    - 3.6
+    - 3.7

NEWS

@@ -1,3 +1,17 @@
+1.2.10
+ * #105: Support for Borg --chunker-params create option via "chunker_params" in borgmatic's storage
+   section.
+
+1.2.9
+ * #102: Fix for syntax error that occurred in Python 3.5 and below.
+ * Make automated tests support running in Python 3.5.
+
+1.2.8
+ * #73: Enable consistency checks for only certain repositories via "check_repositories" option in
+   borgmatic's consistency configuration. Handy for large repositories that take forever to check.
+ * Include link to issue tracker within various command output.
+ * Run continuous integration tests on a matrix of Python and Borg versions.
+
 1.2.7
  * #98: Support for Borg --keep-secondly prune option.
  * Use Black code formatter and Flake8 code checker as part of running automated tests.

README.md

@@ -56,13 +56,17 @@ href="https://asciinema.org/a/203761" target="_blank">screencast</a>.
 
 ## Installation
 
-To get up and running, follow the [Borg Quick
+To get up and running, first [install
+Borg](https://borgbackup.readthedocs.io/en/latest/installation.html), at
+least version 1.1. Then, follow the [Borg Quick
 Start](https://borgbackup.readthedocs.org/en/latest/quickstart.html) to create
-a repository on a local or remote host. Note that if you plan to run borgmatic
-on a schedule with cron, and you encrypt your Borg repository with a
-passphrase instead of a key file, you'll either need to set the borgmatic
-`encryption_passphrase` configuration variable or set the `BORG_PASSPHRASE`
-environment variable. See the [repository encryption
+a repository on a local or remote host.
+
+Note that if you plan to run borgmatic on a schedule with cron, and you
+encrypt your Borg repository with a passphrase instead of a key file, you'll
+either need to set the borgmatic `encryption_passphrase` configuration
+variable or set the `BORG_PASSPHRASE` environment variable. See the
+[repository encryption
 section](https://borgbackup.readthedocs.io/en/latest/quickstart.html#repository-encryption)
 of the Quick Start for more info.
 
@@ -441,9 +445,6 @@ cd borgmatic
 tox
 ```
 
-Note that while running borgmatic itself only requires Python 3+, running
-borgmatic's tests require Python 3.6+.
-
 If when running tests, you get an error from the
 [Black](https://black.readthedocs.io/en/stable/) code formatter about files
 that would be reformatted, you can ask Black to format them for you via the
@@ -478,7 +479,7 @@ consistency:
 ```
 
 This error can be caused by an ssh timeout, which you can rectify by adding
-the following to the ~/.ssh/config file on the client:
+the following to the `~/.ssh/config` file on the client:
 
 ```text
     Host *

borgmatic/borg/create.py

@@ -115,6 +115,7 @@ def create_archive(
     pattern_file = _write_pattern_file(location_config.get('patterns'))
     exclude_file = _write_pattern_file(_expand_directories(location_config.get('exclude_patterns')))
     checkpoint_interval = storage_config.get('checkpoint_interval', None)
+    chunker_params = storage_config.get('chunker_params', None)
     compression = storage_config.get('compression', None)
     remote_rate_limit = storage_config.get('remote_rate_limit', None)
     umask = storage_config.get('umask', None)
@@ -135,6 +136,7 @@ def create_archive(
         + _make_pattern_flags(location_config, pattern_file.name if pattern_file else None)
         + _make_exclude_flags(location_config, exclude_file.name if exclude_file else None)
         + (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ())
+        + (('--chunker-params', chunker_params) if chunker_params else ())
         + (('--compression', compression) if compression else ())
         + (('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ())
         + (('--one-file-system',) if location_config.get('one_file_system') else ())

borgmatic/commands/borgmatic.py

@@ -13,7 +13,7 @@ from borgmatic.borg import (
     info as borg_info,
 )
 from borgmatic.commands import hook
-from borgmatic.config import collect, convert, validate
+from borgmatic.config import checks, collect, convert, validate
 from borgmatic.signals import configure_signals
 from borgmatic.verbosity import verbosity_to_log_level
 
@@ -144,7 +144,15 @@ def run_configuration(config_filename, args):  # pragma: no cover
         if args.create:
             hook.execute_hook(hooks.get('before_backup'), config_filename, 'pre-backup')
 
-        _run_commands(args, consistency, local_path, location, remote_path, retention, storage)
+        _run_commands(
+            args=args,
+            consistency=consistency,
+            local_path=local_path,
+            location=location,
+            remote_path=remote_path,
+            retention=retention,
+            storage=storage,
+        )
 
         if args.create:
             hook.execute_hook(hooks.get('after_backup'), config_filename, 'post-backup')
@@ -153,25 +161,26 @@ def run_configuration(config_filename, args):  # pragma: no cover
         raise
 
 
-def _run_commands(args, consistency, local_path, location, remote_path, retention, storage):
+def _run_commands(*, args, consistency, local_path, location, remote_path, retention, storage):
     json_results = []
     for unexpanded_repository in location['repositories']:
         _run_commands_on_repository(
-            args,
-            consistency,
-            json_results,
-            local_path,
-            location,
-            remote_path,
-            retention,
-            storage,
-            unexpanded_repository,
+            args=args,
+            consistency=consistency,
+            json_results=json_results,
+            local_path=local_path,
+            location=location,
+            remote_path=remote_path,
+            retention=retention,
+            storage=storage,
+            unexpanded_repository=unexpanded_repository,
         )
     if args.json:
         sys.stdout.write(json.dumps(json_results))
 
 
 def _run_commands_on_repository(
+    *,
     args,
     consistency,
     json_results,
@@ -180,7 +189,7 @@ def _run_commands_on_repository(
     remote_path,
     retention,
     storage,
-    unexpanded_repository,
+    unexpanded_repository
 ):  # pragma: no cover
     repository = os.path.expanduser(unexpanded_repository)
     dry_run_label = ' (dry run; not making any changes)' if args.dry_run else ''
@@ -204,7 +213,7 @@ def _run_commands_on_repository(
             local_path=local_path,
             remote_path=remote_path,
         )
-    if args.check:
+    if args.check and checks.repository_enabled_for_checks(repository, consistency):
         logger.info('{}: Running consistency checks'.format(repository))
         borg_check.check_archives(
             repository, storage, consistency, local_path=local_path, remote_path=remote_path
@@ -248,4 +257,6 @@ def main():  # pragma: no cover
             run_configuration(config_filename, args)
     except (ValueError, OSError, CalledProcessError) as error:
         print(error, file=sys.stderr)
+        print(file=sys.stderr)
+        print('Need some help? https://torsion.org/borgmatic/#issues', file=sys.stderr)
         sys.exit(1)

borgmatic/commands/generate_config.py

@@ -38,6 +38,8 @@ def main():  # pragma: no cover
         print()
         print('Please edit the file to suit your needs. The values are just representative.')
         print('All fields are optional except where indicated.')
+        print()
+        print('If you ever need help: https://torsion.org/borgmatic/#issues')
     except (ValueError, OSError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)

borgmatic/config/checks.py

@@ -0,0 +1,9 @@
+def repository_enabled_for_checks(repository, consistency):
+    '''
+    Given a repository name and a consistency configuration dict, return whether the repository
+    is enabled to have consistency checks run.
+    '''
+    if not consistency.get('check_repositories'):
+        return True
+
+    return repository in consistency['check_repositories']

borgmatic/config/schema.yaml

@@ -138,6 +138,13 @@ map:
                     https://borgbackup.readthedocs.io/en/stable/faq.html#if-a-backup-stops-mid-way-does-the-already-backed-up-data-stay-there
                     for details. Defaults to checkpoints every 1800 seconds (30 minutes).
                 example: 1800
+            chunker_params:
+                type: scalar
+                desc: |
+                    Specify the parameters passed to then chunker (CHUNK_MIN_EXP, CHUNK_MAX_EXP,
+                    HASH_MASK_BITS, HASH_WINDOW_SIZE). See https://borgbackup.readthedocs.io/en/stable/internals.html
+                    for details.
+                example: 19,23,21,4095
             compression:
                 type: scalar
                 desc: |
@@ -236,6 +243,16 @@ map:
                 example:
                     - repository
                     - archives
+            check_repositories:
+                seq:
+                    - type: scalar
+                desc: |
+                    Paths to a subset of the repositories in the location section on which to run
+                    consistency checks. Handy in case some of your repositories are very large, and
+                    so running consistency checks on them would take too long. Defaults to running
+                    consistency checks on all repositories configured in the location section.
+                example:
+                    - user@backupserver:sourcehostname.borg
             check_last:
                 type: int
                 desc: Restrict the number of checked archives to the last n. Applies only to the

borgmatic/config/validate.py

@@ -51,6 +51,19 @@ def apply_logical_validation(config_filename, parsed_configuration):
             ('If you provide an archive_name_format, you must also specify a retention prefix.',),
         )
 
+    location_repositories = parsed_configuration.get('location', {}).get('repositories')
+    check_repositories = parsed_configuration.get('consistency', {}).get('check_repositories', [])
+    for repository in check_repositories:
+        if repository not in location_repositories:
+            raise Validation_error(
+                config_filename,
+                (
+                    'Unknown repository in the consistency section\'s check_repositories: {}'.format(
+                        repository
+                    ),
+                ),
+            )
+
     consistency_prefix = parsed_configuration.get('consistency', {}).get('prefix')
     if archive_name_format and not consistency_prefix:
         logger.warning(

scripts/black

@@ -0,0 +1,7 @@
+#!/usr/bin/env sh
+
+if which black; then
+    black --skip-string-normalization --line-length 100 --check .
+else
+    echo "Skipping black due to not being installed."
+fi

scripts/release

@@ -2,13 +2,38 @@
 
 set -e
 
+projects_token=${1:-}
+github_token=${2:-}
+
+if [[ -z $github_token ]]; then
+    echo "Usage: $0 [projects-token] [github-token]"
+    exit 1
+fi
+if [[ ! -f NEWS ]]; then
+    echo "Missing NEWS file. Try running from root of repository."
+    exit 1
+fi
+
 version=$(head --lines=1 NEWS)
 git tag $version
 git push origin $version
 git push github $version
 
+# Build borgmatic and publish to pypi.
 rm -fr dist
 python3 setup.py bdist_wheel
 python3 setup.py sdist
 twine upload -r pypi dist/borgmatic-*.tar.gz
 twine upload -r pypi dist/borgmatic-*-py3-none-any.whl
+
+# Set release changelogs on projects.evoworx.org and GitHub.
+release_changelog="$(cat NEWS | sed '/^$/q' | grep -v '^\S')"
+escaped_release_changelog="$(echo "$release_changelog" | sed -z 's/\n/\\n/g' | sed -z 's/\"/\\"/g')"
+curl --silent --request POST \
+    "https://projects.torsion.org/api/v1/repos/witten/borgmatic/releases?access_token=$projects_token" \
+    --header "Accept: application/json" \
+    --header "Content-Type: application/json" \
+    --data "{\"body\": \"$escaped_release_changelog\", \"draft\": false, \"name\": \"borgmatic $version\", \"prerelease\": false, \"tag_name\": \"$version\"}"
+
+github-release create --token="$github_token" --owner=witten --repo=borgmatic --tag="$version" \
+    --name="borgmatic $version" --body="$release_changelog"

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.2.7'
+VERSION = '1.2.10'
 
 
 setup(
@@ -30,6 +30,5 @@ setup(
     },
     obsoletes=['atticmatic'],
     install_requires=('pykwalify>=1.6.0,<14.06', 'ruamel.yaml>0.15.0,<0.16.0', 'setuptools'),
-    tests_require=('flexmock', 'pytest'),
     include_package_data=True,
 )

test_requirements.txt

@@ -1,7 +1,23 @@
-black==18.9b0
+appdirs==1.4.3
+atomicwrites==1.2.1
+attrs==18.2.0
+black==18.9b0; python_version >= '3.6'
+Click==7.0
+coverage==4.5.1
+docopt==0.6.2
 flake8==3.5.0
 flexmock==0.10.2
+mccabe==0.6.1
+more-itertools==4.3.0
+pluggy==0.7.1
+py==1.6.0
+pycodestyle==2.3.1
+pyflakes==2.0.0
 pykwalify==1.7.0
-pytest==3.8.1
+pytest==3.8.2
 pytest-cov==2.6.0
+python-dateutil==2.7.3
+PyYAML==3.13
 ruamel.yaml>0.15.0,<0.16.0
+six==1.11.0
+toml==0.10.0

tests/end-to-end/test_borgmatic.py

@@ -11,12 +11,14 @@ def generate_configuration(config_path, repository_path):
     Generate borgmatic configuration into a file at the config path, and update the defaults so as
     to work for testing (including injecting the given repository path).
     '''
-    subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' '))
+    subprocess.check_call(
+        'generate-borgmatic-config --destination {}'.format(config_path).split(' ')
+    )
     config = (
         open(config_path)
         .read()
         .replace('user@backupserver:sourcehostname.borg', repository_path)
-        .replace('- /home', f'- {config_path}')
+        .replace('- /home', '- {}'.format(config_path))
         .replace('- /etc', '')
         .replace('- /var/log/syslog*', '')
     )
@@ -32,7 +34,7 @@ def test_borgmatic_command():
 
     try:
         subprocess.check_call(
-            f'borg init --encryption repokey {repository_path}'.split(' '),
+            'borg init --encryption repokey {}'.format(repository_path).split(' '),
             env={'BORG_PASSPHRASE': '', **os.environ},
         )
 
@@ -40,14 +42,22 @@ def test_borgmatic_command():
         generate_configuration(config_path, repository_path)
 
         # Run borgmatic to generate a backup archive, and then list it to make sure it exists.
-        subprocess.check_call(f'borgmatic --config {config_path}'.split(' '))
+        subprocess.check_call('borgmatic --config {}'.format(config_path).split(' '))
         output = subprocess.check_output(
-            f'borgmatic --config {config_path} --list --json'.split(' '),
-            encoding=sys.stdout.encoding,
-        )
+            'borgmatic --config {} --list --json'.format(config_path).split(' ')
+        ).decode(sys.stdout.encoding)
         parsed_output = json.loads(output)
 
         assert len(parsed_output) == 1
         assert len(parsed_output[0]['archives']) == 1
+
+        # Also exercise the info flag.
+        output = subprocess.check_output(
+            'borgmatic --config {} --info --json'.format(config_path).split(' ')
+        ).decode(sys.stdout.encoding)
+        parsed_output = json.loads(output)
+
+        assert len(parsed_output) == 1
+        assert 'repository' in parsed_output[0]
     finally:
         shutil.rmtree(temporary_directory)

tests/unit/borg/test_create.py

@@ -390,6 +390,25 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
     )
 
 
+def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_parameters():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND + ('--chunker-params', '1,2,3,4'))
+
+    module.create_archive(
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={'chunker_params': '1,2,3,4'},
+    )
+
+
 def test_create_archive_with_compression_calls_borg_with_compression_parameters():
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)

tests/unit/commands/test_borgmatic.py

@@ -6,7 +6,7 @@ from flexmock import flexmock
 from borgmatic.commands import borgmatic
 
 
-def test__run_commands_handles_multiple_json_outputs_in_array():
+def test_run_commands_handles_multiple_json_outputs_in_array():
     (
         flexmock(borgmatic)
         .should_receive('_run_commands_on_repository')

tests/unit/config/test_checks.py

@@ -0,0 +1,23 @@
+from borgmatic.config import checks as module
+
+
+def test_repository_enabled_for_checks_defaults_to_enabled_for_all_repositories():
+    enabled = module.repository_enabled_for_checks('repo.borg', consistency={})
+
+    assert enabled
+
+
+def test_repository_enabled_for_checks_is_enabled_for_specified_repositories():
+    enabled = module.repository_enabled_for_checks(
+        'repo.borg', consistency={'check_repositories': ['repo.borg', 'other.borg']}
+    )
+
+    assert enabled
+
+
+def test_repository_enabled_for_checks_is_disabled_for_other_repositories():
+    enabled = module.repository_enabled_for_checks(
+        'repo.borg', consistency={'check_repositories': ['other.borg']}
+    )
+
+    assert not enabled

tests/unit/config/test_validate.py

@@ -51,6 +51,29 @@ def test_apply_logical_validation_warns_if_archive_name_format_present_without_c
     )
 
 
+def test_apply_locical_validation_raises_if_unknown_repository_in_check_repositories():
+    with pytest.raises(module.Validation_error):
+        module.apply_logical_validation(
+            'config.yaml',
+            {
+                'location': {'repositories': ['repo.borg', 'other.borg']},
+                'retention': {'keep_secondly': 1000},
+                'consistency': {'check_repositories': ['repo.borg', 'unknown.borg']},
+            },
+        )
+
+
+def test_apply_locical_validation_does_not_raise_if_known_repository_in_check_repositories():
+    module.apply_logical_validation(
+        'config.yaml',
+        {
+            'location': {'repositories': ['repo.borg', 'other.borg']},
+            'retention': {'keep_secondly': 1000},
+            'consistency': {'check_repositories': ['repo.borg']},
+        },
+    )
+
+
 def test_apply_logical_validation_does_not_raise_or_warn_if_archive_name_format_and_prefix_present():
     logger = flexmock(module.logger)
     logger.should_receive('warning').never()

tox.ini

@@ -5,10 +5,11 @@ skipsdist = True
 [testenv]
 usedevelop = True
 deps = -rtest_requirements.txt
+whitelist_externals = sh
 commands =
     py.test --cov-report term-missing:skip-covered --cov=borgmatic --ignore=tests/end-to-end \
         tests []
-    black --skip-string-normalization --line-length 100 --check .
+    sh scripts/black
     flake8 .
 
 [testenv:black]