Update AUTHORS with recent changes + sort.

AUTHORS

@@ -2,7 +2,8 @@ Dan Helfman <witten@torsion.org>: Main developer
 
 Alexander Görtz: Python 3 compatibility
 Henning Schroeder: Copy editing
+Johannes Feichtner: Support for user hooks
 Michele Lazzeri: Custom archive names
 Robin `ypid` Schneider: Support additional options of Borg
 Scott Squires: Custom archive names
-Johannes Feichtner: Support for user hooks
+Thomas LÉVEIL: Support for a keep_minutely prune option

NEWS

@@ -1,3 +1,12 @@
+1.1.12.dev0
+ * #45: Declare dependency on pykwalify 1.6 or above, as older versions yield "Unknown key: version"
+   rule errors.
+
+1.1.11
+ * #25: Add "ssh_command" to configuration for specifying a custom SSH command or options.
+ * Fix for incorrect /etc/borgmatic.d/ configuration path probing on macOS. This problem manifested
+   as an error on startup: "[Errno 2] No such file or directory: '/etc/borgmatic.d'".
+
 1.1.10
  * Pass several Unix signals through to child processes like Borg. This means that Borg now properly
    shuts down if borgmatic is terminated (e.g. due to a system suspend).

README.md

@@ -1,4 +1,4 @@
-<img src="https://projects.torsion.org/witten/borgmatic/raw/master/static/borgmatic.png" width="150px" style="float: right; padding-left: 1em;">
+<img src="https://projects.torsion.org/witten/borgmatic/raw/branch/master/static/borgmatic.png" width="150px" style="float: right; padding-left: 1em;">
 
 ## Overview
 
@@ -49,11 +49,13 @@ on [GitHub](https://github.com/witten/borgmatic) for convenience.
 
 To get up and running, follow the [Borg Quick
 Start](https://borgbackup.readthedocs.org/en/latest/quickstart.html) to create
-a repository on a local or remote host. Note that if you plan to run
-borgmatic on a schedule with cron, and you encrypt your Borg repository with
-a passphrase instead of a key file, you'll need to set the borgmatic
-`encryption_passphrase` configuration variable. See the repository encryption
-section of the Quick Start for more info.
+a repository on a local or remote host. Note that if you plan to run borgmatic
+on a schedule with cron, and you encrypt your Borg repository with a
+passphrase instead of a key file, you'll either need to set the borgmatic
+`encryption_passphrase` configuration variable or set the `BORG_PASSPHRASE`
+environment variable. See the [repository encryption
+section](https://borgbackup.readthedocs.io/en/latest/quickstart.html#repository-encryption)
+of the Quick Start for more info.
 
 If the repository is on a remote host, make sure that your local root user has
 key-based ssh access to the desired user account on the remote host.

borgmatic/borg/create.py

@@ -11,12 +11,15 @@ from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 logger = logging.getLogger(__name__)
 
 
-def initialize(storage_config):
+def initialize_environment(storage_config):
     passphrase = storage_config.get('encryption_passphrase')
-
     if passphrase:
         os.environ['BORG_PASSPHRASE'] = passphrase
 
+    ssh_command = storage_config.get('ssh_command')
+    if ssh_command:
+        os.environ['BORG_RSH'] = ssh_command
+
 
 def _expand_directory(directory):
     '''

borgmatic/commands/borgmatic.py

@@ -94,7 +94,7 @@ def run_configuration(config_filename, args):  # pragma: no cover
 
     try:
         remote_path = location.get('remote_path')
-        create.initialize(storage)
+        create.initialize_environment(storage)
         hook.execute_hook(hooks.get('before_backup'), config_filename, 'pre-backup')
 
         for unexpanded_repository in location['repositories']:

borgmatic/config/collect.py

@@ -11,13 +11,15 @@ def collect_config_filenames(config_paths):
     files. This is non-recursive, so any directories within the given directories are ignored.
 
     Return paths even if they don't exist on disk, so the user can find out about missing
-    configuration paths. However, skip /etc/borgmatic.d if it's missing, so the user doesn't have to
-    create it unless they need it.
+    configuration paths. However, skip a default config path if it's missing, so the user doesn't
+    have to create a default config path unless they need it.
     '''
+    real_default_config_paths = set(map(os.path.realpath, DEFAULT_CONFIG_PATHS))
+
     for path in config_paths:
         exists = os.path.exists(path)
 
-        if os.path.realpath(path) in DEFAULT_CONFIG_PATHS and not exists:
+        if os.path.realpath(path) in real_default_config_paths and not exists:
             continue
 
         if not os.path.isdir(path) or not exists:

borgmatic/config/schema.yaml

@@ -96,6 +96,10 @@ map:
                 type: int
                 desc: Remote network upload rate limit in kiBytes/second.
                 example: 100
+            ssh_command:
+                type: scalar
+                desc: Command to use instead of just "ssh". This can be used to specify ssh options.
+                example: ssh -i /path/to/private/key
             umask:
                 type: scalar
                 desc: Umask to be used for borg create.
@@ -118,6 +122,10 @@ map:
                 type: scalar
                 desc: Keep all archives within this time interval.
                 example: 3H
+            keep_minutely:
+                type: int
+                desc: Number of minutely archives to keep.
+                example: 60
             keep_hourly:
                 type: int
                 desc: Number of hourly archives to keep.

borgmatic/tests/integration/config/test_validate.py

@@ -44,6 +44,8 @@ def test_parse_configuration_transforms_file_into_mapping():
                 - hostname.borg
 
         retention:
+            keep_minutely: 60
+            keep_hourly: 24
             keep_daily: 7
 
         consistency:
@@ -57,7 +59,7 @@ def test_parse_configuration_transforms_file_into_mapping():
 
     assert result == {
         'location': {'source_directories': ['/home', '/etc'], 'repositories': ['hostname.borg']},
-        'retention': {'keep_daily': 7},
+        'retention': {'keep_daily': 7, 'keep_hourly': 24, 'keep_minutely': 60},
         'consistency': {'checks': ['repository', 'archives']},
     }
 

borgmatic/tests/unit/borg/test_create.py

@@ -6,24 +6,36 @@ from borgmatic.borg import create as module
 from borgmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 
 
-def test_initialize_with_passphrase_should_set_environment():
+def test_initialize_environment_with_passphrase_should_set_environment():
     orig_environ = os.environ
 
     try:
         os.environ = {}
-        module.initialize({'encryption_passphrase': 'pass'})
+        module.initialize_environment({'encryption_passphrase': 'pass'})
         assert os.environ.get('BORG_PASSPHRASE') == 'pass'
     finally:
         os.environ = orig_environ
 
 
-def test_initialize_without_passphrase_should_not_set_environment():
+def test_initialize_environment_with_ssh_command_should_set_environment():
     orig_environ = os.environ
 
     try:
         os.environ = {}
-        module.initialize({})
+        module.initialize_environment({'ssh_command': 'ssh -C'})
+        assert os.environ.get('BORG_RSH') == 'ssh -C'
+    finally:
+        os.environ = orig_environ
+
+
+def test_initialize_environment_without_configuration_should_not_set_environment():
+    orig_environ = os.environ
+
+    try:
+        os.environ = {}
+        module.initialize_environment({})
         assert os.environ.get('BORG_PASSPHRASE') == None
+        assert os.environ.get('BORG_RSH') == None
     finally:
         os.environ = orig_environ
 

borgmatic/tests/unit/config/test_collect.py

@@ -58,6 +58,19 @@ def test_collect_config_filenames_skips_etc_borgmatic_dot_d_if_it_does_not_exist
     assert config_filenames == ('config.yaml',)
 
 
+def test_collect_config_filenames_skips_non_canonical_etc_borgmatic_dot_d_if_it_does_not_exist():
+    config_paths = ('config.yaml', '/etc/../etc/borgmatic.d')
+    mock_path = flexmock(module.os.path)
+    mock_path.should_receive('exists').with_args('config.yaml').and_return(True)
+    mock_path.should_receive('exists').with_args('/etc/../etc/borgmatic.d').and_return(False)
+    mock_path.should_receive('isdir').with_args('config.yaml').and_return(False)
+    mock_path.should_receive('isdir').with_args('/etc/../etc/borgmatic.d').and_return(True)
+
+    config_filenames = tuple(module.collect_config_filenames(config_paths))
+
+    assert config_filenames == ('config.yaml',)
+
+
 def test_collect_config_filenames_includes_other_directory_if_it_does_not_exist():
     config_paths = ('config.yaml', '/my/directory')
     mock_path = flexmock(module.os.path)

scripts/find-unsupported-borg-options

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# For each Borg sub-command that borgmatic uses, print out the Borg flags that borgmatic does not
+# appear to support yet. This script isn't terribly robust. It's intended as a basic tool to ferret
+# out unsupported Borg options so that they can be considered for addition to borgmatic.
+
+# Generate a sample borgmatic configuration with all options set.
+generate-borgmatic-config --destination temp.yaml
+
+# For each sub-command (prune, create, and check), collect the Borg command-line flags that result
+# from running borgmatic with the generated configuration. Then, collect the full set of available
+# Borg flags as reported by "borg --help" for that sub-command. Finally, compare the two lists of
+# flags to determine which Borg flags borgmatic doesn't yet support.
+for sub_command in prune create check; do
+    echo "********** borg $sub_command **********"
+
+    for line in $(borgmatic --config temp.yaml --$sub_command -v 2 2>&1 | grep "borg $sub_command") ; do
+        echo "$line" | grep '^-' >> borgmatic_borg_flags
+    done
+    sort borgmatic_borg_flags > borgmatic_borg_flags.sorted
+    mv borgmatic_borg_flags.sorted borgmatic_borg_flags
+
+    for line in $(borg $sub_command --help | awk -v RS= '/^usage:/') ; do
+        # Exclude a bunch of flags that borgmatic actually supports, but don't get exercised by the
+        # generated sample config, and also flags that don't make sense to support.
+        echo "$line" | grep -- -- | sed -r 's/(\[|\])//g' \
+            | grep -v '^-h$' \
+            | grep -v '^--archives-only$' \
+            | grep -v '^--repository-only$' \
+            | grep -v '^--stats$' \
+            | grep -v '^--list$' \
+            | grep -v '^--critical$' \
+            | grep -v '^--error$' \
+            | grep -v '^--warning$' \
+            | grep -v '^--info$' \
+            | grep -v '^--debug$' \
+            >> all_borg_flags
+    done
+    sort all_borg_flags > all_borg_flags.sorted
+    mv all_borg_flags.sorted all_borg_flags
+
+    comm -13 borgmatic_borg_flags all_borg_flags
+
+    rm ./*_borg_flags
+done
+
+rm temp.yaml

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '1.1.10'
+VERSION = '1.1.12.dev0'
 
 
 setup(
@@ -32,7 +32,7 @@ setup(
         'atticmatic',
     ],
     install_requires=(
-        'pykwalify',
+        'pykwalify>=1.6.0',
         'ruamel.yaml<=0.15',
         'setuptools',
     ),