Support for Borg create --progress via borgmatic command-line flag (#108).

.drone.yml

@@ -13,5 +13,6 @@ matrix:
     - 3.7
     - 3.8
   PYTHON_VERSION:
+    - 3.5
     - 3.6
     - 3.7

NEWS

@@ -1,3 +1,14 @@
+1.2.11
+ * #108: Support for Borg create --progress via borgmatic command-line flag.
+
+1.2.10
+ * #105: Support for Borg --chunker-params create option via "chunker_params" in borgmatic's storage
+   section.
+
+1.2.9
+ * #102: Fix for syntax error that occurred in Python 3.5 and below.
+ * Make automated tests support running in Python 3.5.
+
 1.2.8
  * #73: Enable consistency checks for only certain repositories via "check_repositories" option in
    borgmatic's consistency configuration. Handy for large repositories that take forever to check.

README.md

@@ -445,9 +445,6 @@ cd borgmatic
 tox
 ```
 
-Note that while running borgmatic itself only requires Python 3+, running
-borgmatic's tests require Python 3.6+.
-
 If when running tests, you get an error from the
 [Black](https://black.readthedocs.io/en/stable/) code formatter about files
 that would be reformatted, you can ask Black to format them for you via the

borgmatic/borg/create.py

@@ -104,6 +104,7 @@ def create_archive(
     storage_config,
     local_path='borg',
     remote_path=None,
+    progress=False,
     json=False,
 ):
     '''
@@ -115,6 +116,7 @@ def create_archive(
     pattern_file = _write_pattern_file(location_config.get('patterns'))
     exclude_file = _write_pattern_file(_expand_directories(location_config.get('exclude_patterns')))
     checkpoint_interval = storage_config.get('checkpoint_interval', None)
+    chunker_params = storage_config.get('chunker_params', None)
     compression = storage_config.get('compression', None)
     remote_rate_limit = storage_config.get('remote_rate_limit', None)
     umask = storage_config.get('umask', None)
@@ -135,6 +137,7 @@ def create_archive(
         + _make_pattern_flags(location_config, pattern_file.name if pattern_file else None)
         + _make_exclude_flags(location_config, exclude_file.name if exclude_file else None)
         + (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ())
+        + (('--chunker-params', chunker_params) if chunker_params else ())
         + (('--compression', compression) if compression else ())
         + (('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ())
         + (('--one-file-system',) if location_config.get('one_file_system') else ())
@@ -149,6 +152,7 @@ def create_archive(
         + (('--stats',) if not dry_run and logger.isEnabledFor(logging.INFO) else ())
         + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
         + (('--dry-run',) if dry_run else ())
+        + (('--progress',) if progress else ())
         + (('--json',) if json else ())
     )
 

borgmatic/commands/borgmatic.py

@@ -78,6 +78,13 @@ def parse_arguments(*arguments):
         action='store_true',
         help='Display summary information on archives',
     )
+    parser.add_argument(
+        '--progress',
+        dest='progress',
+        default=False,
+        action='store_true',
+        help='Display progress with --create option for each file as it is backed up',
+    )
     parser.add_argument(
         '--json',
         dest='json',
@@ -103,6 +110,9 @@ def parse_arguments(*arguments):
 
     args = parser.parse_args(arguments)
 
+    if args.progress and not args.create:
+        raise ValueError('The --progress option can only be used with the --create option')
+
     if args.json and not (args.create or args.list or args.info):
         raise ValueError(
             'The --json option can only be used with the --create, --list, or --info options'
@@ -189,7 +199,7 @@ def _run_commands_on_repository(
     remote_path,
     retention,
     storage,
-    unexpanded_repository,
+    unexpanded_repository
 ):  # pragma: no cover
     repository = os.path.expanduser(unexpanded_repository)
     dry_run_label = ' (dry run; not making any changes)' if args.dry_run else ''
@@ -212,6 +222,7 @@ def _run_commands_on_repository(
             storage,
             local_path=local_path,
             remote_path=remote_path,
+            progress=args.progress,
         )
     if args.check and checks.repository_enabled_for_checks(repository, consistency):
         logger.info('{}: Running consistency checks'.format(repository))

borgmatic/config/schema.yaml

@@ -138,6 +138,13 @@ map:
                     https://borgbackup.readthedocs.io/en/stable/faq.html#if-a-backup-stops-mid-way-does-the-already-backed-up-data-stay-there
                     for details. Defaults to checkpoints every 1800 seconds (30 minutes).
                 example: 1800
+            chunker_params:
+                type: scalar
+                desc: |
+                    Specify the parameters passed to then chunker (CHUNK_MIN_EXP, CHUNK_MAX_EXP,
+                    HASH_MASK_BITS, HASH_WINDOW_SIZE). See https://borgbackup.readthedocs.io/en/stable/internals.html
+                    for details.
+                example: 19,23,21,4095
             compression:
                 type: scalar
                 desc: |

scripts/black

@@ -0,0 +1,7 @@
+#!/usr/bin/env sh
+
+if which black; then
+    black --skip-string-normalization --line-length 100 --check .
+else
+    echo "Skipping black due to not being installed."
+fi

scripts/find-unsupported-borg-options

@@ -40,6 +40,7 @@ for sub_command in prune create check list info; do
             | grep -v '^--list$' \
             | grep -v '^--nobsdflags$' \
             | grep -v '^--pattern$' \
+            | grep -v '^--progress$' \
             | grep -v '^--read-special$' \
             | grep -v '^--repository-only$' \
             | grep -v '^--show-rc$' \

scripts/release

@@ -27,9 +27,9 @@ twine upload -r pypi dist/borgmatic-*.tar.gz
 twine upload -r pypi dist/borgmatic-*-py3-none-any.whl
 
 # Set release changelogs on projects.evoworx.org and GitHub.
-release_changelog=$(sed '/^$/q' NEWS |grep '^\s*\*')
+release_changelog="$(cat NEWS | sed '/^$/q' | grep -v '^\S')"
-escaped_release_changelog=$(echo $release_changelog | sed -z 's/\n/\\n/g')
+escaped_release_changelog="$(echo "$release_changelog" | sed -z 's/\n/\\n/g' | sed -z 's/\"/\\"/g')"
-curl --request POST \
+curl --silent --request POST \
     "https://projects.torsion.org/api/v1/repos/witten/borgmatic/releases?access_token=$projects_token" \
     --header "Accept: application/json" \
     --header "Content-Type: application/json" \

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.2.8'
+VERSION = '1.2.11'
 
 
 setup(
@@ -30,6 +30,5 @@ setup(
     },
     obsoletes=['atticmatic'],
     install_requires=('pykwalify>=1.6.0,<14.06', 'ruamel.yaml>0.15.0,<0.16.0', 'setuptools'),
-    tests_require=('flexmock', 'pytest'),
     include_package_data=True,
 )

test_requirements.txt

@@ -1,7 +1,7 @@
 appdirs==1.4.3
 atomicwrites==1.2.1
 attrs==18.2.0
-black==18.9b0
+black==18.9b0; python_version >= '3.6'
 Click==7.0
 coverage==4.5.1
 docopt==0.6.2

tests/end-to-end/test_borgmatic.py

@@ -11,12 +11,14 @@ def generate_configuration(config_path, repository_path):
     Generate borgmatic configuration into a file at the config path, and update the defaults so as
     to work for testing (including injecting the given repository path).
     '''
-    subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' '))
+    subprocess.check_call(
+        'generate-borgmatic-config --destination {}'.format(config_path).split(' ')
+    )
     config = (
         open(config_path)
         .read()
         .replace('user@backupserver:sourcehostname.borg', repository_path)
-        .replace('- /home', f'- {config_path}')
+        .replace('- /home', '- {}'.format(config_path))
         .replace('- /etc', '')
         .replace('- /var/log/syslog*', '')
     )
@@ -32,7 +34,7 @@ def test_borgmatic_command():
 
     try:
         subprocess.check_call(
-            f'borg init --encryption repokey {repository_path}'.split(' '),
+            'borg init --encryption repokey {}'.format(repository_path).split(' '),
             env={'BORG_PASSPHRASE': '', **os.environ},
         )
 
@@ -40,11 +42,10 @@ def test_borgmatic_command():
         generate_configuration(config_path, repository_path)
 
         # Run borgmatic to generate a backup archive, and then list it to make sure it exists.
-        subprocess.check_call(f'borgmatic --config {config_path}'.split(' '))
+        subprocess.check_call('borgmatic --config {}'.format(config_path).split(' '))
         output = subprocess.check_output(
-            f'borgmatic --config {config_path} --list --json'.split(' '),
+            'borgmatic --config {} --list --json'.format(config_path).split(' ')
-            encoding=sys.stdout.encoding,
+        ).decode(sys.stdout.encoding)
-        )
         parsed_output = json.loads(output)
 
         assert len(parsed_output) == 1
@@ -52,9 +53,8 @@ def test_borgmatic_command():
 
         # Also exercise the info flag.
         output = subprocess.check_output(
-            f'borgmatic --config {config_path} --info --json'.split(' '),
+            'borgmatic --config {} --info --json'.format(config_path).split(' ')
-            encoding=sys.stdout.encoding,
+        ).decode(sys.stdout.encoding)
-        )
         parsed_output = json.loads(output)
 
         assert len(parsed_output) == 1

tests/unit/borg/test_create.py

@@ -390,6 +390,25 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte
     )
 
 
+def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_parameters():
+    flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
+    flexmock(module).should_receive('_write_pattern_file').and_return(None)
+    flexmock(module).should_receive('_make_pattern_flags').and_return(())
+    flexmock(module).should_receive('_make_exclude_flags').and_return(())
+    insert_subprocess_mock(CREATE_COMMAND + ('--chunker-params', '1,2,3,4'))
+
+    module.create_archive(
+        dry_run=False,
+        repository='repo',
+        location_config={
+            'source_directories': ['foo', 'bar'],
+            'repositories': ['repo'],
+            'exclude_patterns': None,
+        },
+        storage_config={'chunker_params': '1,2,3,4'},
+    )
+
+
 def test_create_archive_with_compression_calls_borg_with_compression_parameters():
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)

tox.ini

@@ -5,10 +5,11 @@ skipsdist = True
 [testenv]
 usedevelop = True
 deps = -rtest_requirements.txt
+whitelist_externals = sh
 commands =
     py.test --cov-report term-missing:skip-covered --cov=borgmatic --ignore=tests/end-to-end \
         tests []
-    black --skip-string-normalization --line-length 100 --check .
+    sh scripts/black
     flake8 .
 
 [testenv:black]