Revving version.

.hgignore

@@ -3,3 +3,5 @@ syntax: glob
 *.pyc
 *.swp
 .tox
+build
+dist

.hgtags

@@ -14,3 +14,10 @@ cf4c7065f0711deda1cba878398bc05390e2c3f9 0.0.7
 ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1
 ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1
 7b6c87dca7ea312b2257ac1b46857b3f8c56b39c 0.1.1
+83067f995dd391e38544a7722dc3b254b59c5521 0.1.2
+acc7fb61566fe8028c179f43ecc735c851220b06 0.1.3
+6dda59c12de88f060eb7244e6d330173985a9639 0.1.4
+6dda59c12de88f060eb7244e6d330173985a9639 0.1.4
+e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4
+0afff209b902698c2266986129d6dc9f5f913101 0.1.5
+4c63f3d90ec2bf6af1714a3acec84654a7c9edf3 0.1.6

NEWS

@@ -1,3 +1,25 @@
+0.1.7
+
+ * #11: Fixed parsing of punctuation in configuration file.
+ * Better error message when configuration file is missing.
+
+0.1.6
+
+ * #9: New configuration option for the encryption passphrase.
+ * #10: Support for Borg's new archive compression feature.
+
+0.1.5
+
+ * Changes to support release on PyPI. Now pip installable by name!
+
+0.1.4
+
+ * Adding test that setup.py version matches release version.
+
+0.1.3
+
+ * #1: Add support for "borg check --last N" to Borg backend.
+
 0.1.2
 
  * As a convenience to new users, allow a missing default excludes file.

README.md

@@ -1,6 +1,4 @@
 title: Atticmatic
-date:
-save_as: atticmatic/index.html
 
 ## Overview
 
@@ -47,16 +45,16 @@ Start](https://attic-backup.org/quickstart.html) or the [Borg Quick
 Start](https://borgbackup.github.io/borgbackup/quickstart.html) to create a
 repository on a local or remote host. Note that if you plan to run atticmatic
 on a schedule with cron, and you encrypt your attic repository with a
-passphrase instead of a key file, you'll need to set the `ATTIC_PASSPHRASE`
-environment variable. See the repository encryption section of the Quick Start
-for more info.
+passphrase instead of a key file, you'll need to set the atticmatic
+`encryption_passphrase` configuration variable. See the repository encryption
+section of the Quick Start for more info.
 
 If the repository is on a remote host, make sure that your local root user has
 key-based ssh access to the desired user account on the remote host.
 
 To install atticmatic, run the following command to download and install it:
 
-    sudo pip install --upgrade hg+https://torsion.org/hg/atticmatic
+    sudo pip install --upgrade atticmatic
 
 If you are using Attic, copy the following configuration files:
 
@@ -138,6 +136,8 @@ backups.
 ## Issues and feedback
 
 Got an issue or an idea for a feature enhancement? Check out the [atticmatic
-issue tracker](https://tree.taiga.io/project/witten-atticmatic/issues).
+issue tracker](https://tree.taiga.io/project/witten-atticmatic/issues). In
+order to create a new issue or comment on an issue, you'll need to [login
+first](https://tree.taiga.io/login).
 
 Other questions or comments? Contact <mailto:witten@torsion.org>.

atticmatic/backends/attic.py

@@ -5,8 +5,10 @@ from atticmatic.backends import shared
 # An atticmatic backend that supports Attic for actually handling backups.
 
 COMMAND = 'attic'
+CONFIG_FORMAT = shared.CONFIG_FORMAT
 
 
+initialize = partial(shared.initialize, command=COMMAND)
 create_archive = partial(shared.create_archive, command=COMMAND)
 prune_archives = partial(shared.prune_archives, command=COMMAND)
 check_archives = partial(shared.check_archives, command=COMMAND)

atticmatic/backends/borg.py

@@ -1,13 +1,32 @@
 from functools import partial
 
+from atticmatic.config import Section_format, option
 from atticmatic.backends import shared
 
 # An atticmatic backend that supports Borg for actually handling backups.
 
 COMMAND = 'borg'
+CONFIG_FORMAT = (
+    shared.CONFIG_FORMAT[0],  # location
+    Section_format(
+        'storage',
+        (
+            option('encryption_passphrase', required=False),
+            option('compression', required=False),
+        ),
+    ),
+    shared.CONFIG_FORMAT[2],  # retention
+    Section_format(
+        'consistency',
+        (
+            option('checks', required=False),
+            option('check_last', required=False),
+        ),
+    )
+)
 
 
+initialize = partial(shared.initialize, command=COMMAND)
 create_archive = partial(shared.create_archive, command=COMMAND)
 prune_archives = partial(shared.prune_archives, command=COMMAND)
 check_archives = partial(shared.check_archives, command=COMMAND)
-

atticmatic/backends/shared.py

@@ -3,6 +3,7 @@ import os
 import platform
 import subprocess
 
+from atticmatic.config import Section_format, option
 from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 
 
@@ -12,13 +13,60 @@ from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 # atticmatic.backends.borg.
 
 
-def create_archive(excludes_filename, verbosity, source_directories, repository, command):
+CONFIG_FORMAT = (
+    Section_format(
+        'location',
+        (
+            option('source_directories'),
+            option('repository'),
+        ),
+    ),
+    Section_format(
+        'storage',
+        (
+            option('encryption_passphrase', required=False),
+        ),
+    ),
+    Section_format(
+        'retention',
+        (
+            option('keep_within', required=False),
+            option('keep_hourly', int, required=False),
+            option('keep_daily', int, required=False),
+            option('keep_weekly', int, required=False),
+            option('keep_monthly', int, required=False),
+            option('keep_yearly', int, required=False),
+            option('prefix', required=False),
+        ),
+    ),
+    Section_format(
+        'consistency',
+        (
+            option('checks', required=False),
+        ),
+    )
+)
+
+
+def initialize(storage_config, command):
+    passphrase = storage_config.get('encryption_passphrase')
+
+    if passphrase:
+        os.environ['{}_PASSPHRASE'.format(command.upper())] = passphrase
+
+
+def create_archive(
+    excludes_filename, verbosity, storage_config, source_directories, repository, command,
+):
     '''
-    Given an excludes filename (or None), a vebosity flag, a space-separated list of source
-    directories, a local or remote repository path, and a command to run, create an attic archive.
+    Given an excludes filename (or None), a vebosity flag, a storage config dict, a space-separated
+    list of source directories, a local or remote repository path, and a command to run, create an
+    attic archive.
     '''
     sources = tuple(source_directories.split(' '))
     exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else ()
+    compression = storage_config.get('compression', None)
+    compression_flags = ('--compression', compression) if compression else ()
     verbosity_flags = {
         VERBOSITY_SOME: ('--stats',),
         VERBOSITY_LOTS: ('--verbose', '--stats'),
@@ -31,7 +79,7 @@ def create_archive(excludes_filename, verbosity, source_directories, repository,
             hostname=platform.node(),
             timestamp=datetime.now().isoformat(),
         ),
-    ) + sources + exclude_flags + verbosity_flags
+    ) + sources + exclude_flags + compression_flags + verbosity_flags
 
     subprocess.check_call(full_command)
 
@@ -110,7 +158,7 @@ def _parse_checks(consistency_config):
     )
 
 
-def _make_check_flags(checks):
+def _make_check_flags(checks, check_last=None):
     '''
     Given a parsed sequence of checks, transform it into tuple of command-line flags.
 
@@ -121,13 +169,17 @@ def _make_check_flags(checks):
     This will be returned as:
     
         ('--repository-only',)
+
+    Additionally, if a check_last value is given, a "--last" flag will be added. Note that only
+    Borg supports this flag.
     '''
+    last_flag = ('--last', check_last) if check_last else ()
     if checks == DEFAULT_CHECKS:
-        return ()
+        return last_flag
 
     return tuple(
         '--{}-only'.format(check) for check in checks
-    )
+    ) + last_flag
 
 
 def check_archives(verbosity, repository, consistency_config, command):
@@ -138,6 +190,7 @@ def check_archives(verbosity, repository, consistency_config, command):
     If there are no consistency checks to run, skip running them.
     '''
     checks = _parse_checks(consistency_config)
+    check_last = consistency_config.get('check_last', None)
     if not checks:
         return
 
@@ -149,7 +202,7 @@ def check_archives(verbosity, repository, consistency_config, command):
     full_command = (
         command, 'check',
         repository,
-    ) + _make_check_flags(checks) + verbosity_flags
+    ) + _make_check_flags(checks, check_last) + verbosity_flags
 
     # The check command spews to stdout even without the verbose flag. Suppress it.
     stdout = None if verbosity_flags else open(os.devnull, 'w')

atticmatic/command.py

@@ -60,11 +60,14 @@ def main():
     try:
         command_name = os.path.basename(sys.argv[0])
         args = parse_arguments(command_name, *sys.argv[1:])
-        config = parse_configuration(args.config_filename)
-        repository = config.location['repository']
         backend = load_backend(command_name)
+        config = parse_configuration(args.config_filename, backend.CONFIG_FORMAT)
+        repository = config.location['repository']
 
-        backend.create_archive(args.excludes_filename, args.verbosity, **config.location)
+        backend.initialize(config.storage)
+        backend.create_archive(
+            args.excludes_filename, args.verbosity, config.storage, **config.location
+        )
         backend.prune_archives(args.verbosity, repository, config.retention)
         backend.check_archives(args.verbosity, repository, config.consistency)
     except (ValueError, IOError, CalledProcessError) as error:

atticmatic/config.py

@@ -2,10 +2,10 @@ from collections import OrderedDict, namedtuple
 
 try:
     # Python 2
-    from ConfigParser import ConfigParser
+    from ConfigParser import RawConfigParser
 except ImportError:
     # Python 3
-    from configparser import ConfigParser
+    from configparser import RawConfigParser
 
 
 Section_format = namedtuple('Section_format', ('name', 'options'))
@@ -20,38 +20,9 @@ def option(name, value_type=str, required=True):
     return Config_option(name, value_type, required)
 
 
-CONFIG_FORMAT = (
-    Section_format(
-        'location',
-        (
-            option('source_directories'),
-            option('repository'),
-        ),
-    ),
-    Section_format(
-        'retention',
-        (
-            option('keep_within', required=False),
-            option('keep_hourly', int, required=False),
-            option('keep_daily', int, required=False),
-            option('keep_weekly', int, required=False),
-            option('keep_monthly', int, required=False),
-            option('keep_yearly', int, required=False),
-            option('prefix', required=False),
-        ),
-    ),
-    Section_format(
-        'consistency',
-        (
-            option('checks', required=False),
-        ),
-    )
-)
-
-
 def validate_configuration_format(parser, config_format):
     '''
-    Given an open ConfigParser and an expected config file format, validate that the parsed
+    Given an open RawConfigParser and an expected config file format, validate that the parsed
     configuration file has the expected sections, that any required options are present in those
     sections, and that there aren't any unexpected options.
 
@@ -110,14 +81,9 @@ def validate_configuration_format(parser, config_format):
             )
 
 
-# Describes a parsed configuration, where each attribute is the name of a configuration file section 
-# and each value is a dict of that section's parsed options.
-Parsed_config = namedtuple('Config', (section_format.name for section_format in CONFIG_FORMAT))
-
-
 def parse_section_options(parser, section_format):
     '''
-    Given an open ConfigParser and an expected section format, return the option values from that
+    Given an open RawConfigParser and an expected section format, return the option values from that
     section as a dict mapping from option name to value. Omit those options that are not present in
     the parsed options.
 
@@ -135,21 +101,26 @@ def parse_section_options(parser, section_format):
     )
 
 
-def parse_configuration(config_filename):
+def parse_configuration(config_filename, config_format):
     '''
-    Given a config filename of the expected format, return the parsed configuration as Parsed_config
-    data structure.
+    Given a config filename and an expected config file format, return the parsed configuration
+    as a namedtuple with one attribute for each parsed section.
 
     Raise IOError if the file cannot be read, or ValueError if the format is not as expected.
     '''
-    parser = ConfigParser()
-    parser.read(config_filename)
+    parser = RawConfigParser()
+    if not parser.read(config_filename):
+        raise ValueError('Configuration file cannot be opened: {}'.format(config_filename))
 
-    validate_configuration_format(parser, CONFIG_FORMAT)
+    validate_configuration_format(parser, config_format)
+
+    # Describes a parsed configuration, where each attribute is the name of a configuration file
+    # section and each value is a dict of that section's parsed options.
+    Parsed_config = namedtuple('Parsed_config', (section_format.name for section_format in config_format))
 
     return Parsed_config(
         *(
             parse_section_options(parser, section_format)
-            for section_format in CONFIG_FORMAT
+            for section_format in config_format
         )
     )

atticmatic/tests/integration/test_config.py

@@ -0,0 +1,29 @@
+try:
+    # Python 2
+    from cStringIO import StringIO
+except ImportError:
+    # Python 3
+    from io import StringIO
+
+from collections import OrderedDict
+import string
+
+from atticmatic import config as module
+
+
+def test_parse_section_options_with_punctuation_should_return_section_options():
+    parser = module.RawConfigParser()
+    parser.readfp(StringIO('[section]\nfoo: {}\n'.format(string.punctuation)))
+
+    section_format = module.Section_format(
+        'section',
+        (module.Config_option('foo', str, required=True),),
+    )
+
+    config = module.parse_section_options(parser, section_format)
+
+    assert config == OrderedDict(
+        (
+            ('foo', string.punctuation),
+        )
+    )

atticmatic/tests/integration/test_version.py

@@ -0,0 +1,8 @@
+import subprocess
+
+
+def test_setup_version_matches_news_version():
+    setup_version = subprocess.check_output(('python', 'setup.py', '--version')).decode('ascii')
+    news_version = open('NEWS').readline()
+
+    assert setup_version == news_version

atticmatic/tests/unit/backends/test_shared.py

@@ -1,4 +1,5 @@
 from collections import OrderedDict
+import os
 
 from flexmock import flexmock
 
@@ -7,6 +8,28 @@ from atticmatic.tests.builtins import builtins_mock
 from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 
 
+def test_initialize_with_passphrase_should_set_environment():
+    orig_environ = os.environ
+
+    try:
+        os.environ = {}
+        module.initialize({'encryption_passphrase': 'pass'}, command='attic')
+        assert os.environ.get('ATTIC_PASSPHRASE') == 'pass'
+    finally:
+        os.environ = orig_environ
+
+
+def test_initialize_without_passphrase_should_not_set_environment():
+    orig_environ = os.environ
+
+    try:
+        os.environ = {}
+        module.initialize({}, command='attic')
+        assert os.environ.get('ATTIC_PASSPHRASE') == None
+    finally:
+        os.environ = orig_environ
+
+
 def insert_subprocess_mock(check_call_command, **kwargs):
     subprocess = flexmock()
     subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
@@ -41,6 +64,7 @@ def test_create_archive_should_call_attic_with_parameters():
     module.create_archive(
         excludes_filename='excludes',
         verbosity=None,
+        storage_config={},
         source_directories='foo bar',
         repository='repo',
         command='attic',
@@ -55,6 +79,7 @@ def test_create_archive_with_none_excludes_filename_should_call_attic_without_ex
     module.create_archive(
         excludes_filename=None,
         verbosity=None,
+        storage_config={},
         source_directories='foo bar',
         repository='repo',
         command='attic',
@@ -69,6 +94,7 @@ def test_create_archive_with_verbosity_some_should_call_attic_with_stats_paramet
     module.create_archive(
         excludes_filename='excludes',
         verbosity=VERBOSITY_SOME,
+        storage_config={},
         source_directories='foo bar',
         repository='repo',
         command='attic',
@@ -83,6 +109,22 @@ def test_create_archive_with_verbosity_lots_should_call_attic_with_verbose_param
     module.create_archive(
         excludes_filename='excludes',
         verbosity=VERBOSITY_LOTS,
+        storage_config={},
+        source_directories='foo bar',
+        repository='repo',
+        command='attic',
+    )
+
+
+def test_create_archive_with_compression_should_call_attic_with_compression_parameters():
+    insert_subprocess_mock(CREATE_COMMAND + ('--compression', 'rle'))
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        excludes_filename='excludes',
+        verbosity=None,
+        storage_config={'compression': 'rle'},
         source_directories='foo bar',
         repository='repo',
         command='attic',
@@ -196,10 +238,24 @@ def test_make_check_flags_with_default_checks_returns_no_flags():
     assert flags == ()
 
 
+def test_make_check_flags_with_checks_and_last_returns_flags_including_last():
+    flags = module._make_check_flags(('foo', 'bar'), check_last=3)
+
+    assert flags == ('--foo-only', '--bar-only', '--last', 3)
+
+
+def test_make_check_flags_with_last_returns_last_flag():
+    flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3)
+
+    assert flags == ('--last', 3)
+
+
 def test_check_archives_should_call_attic_with_parameters():
-    consistency_config = flexmock()
-    flexmock(module).should_receive('_parse_checks').and_return(flexmock())
-    flexmock(module).should_receive('_make_check_flags').and_return(())
+    checks = flexmock()
+    check_last = flexmock()
+    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
+    flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
     stdout = flexmock()
     insert_subprocess_mock(
         ('attic', 'check', 'repo'),
@@ -219,7 +275,7 @@ def test_check_archives_should_call_attic_with_parameters():
 
 
 def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_parameter():
-    consistency_config = flexmock()
+    consistency_config = flexmock().should_receive('get').and_return(None).mock
     flexmock(module).should_receive('_parse_checks').and_return(flexmock())
     flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_subprocess_mock(
@@ -238,7 +294,7 @@ def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_param
 
 
 def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter():
-    consistency_config = flexmock()
+    consistency_config = flexmock().should_receive('get').and_return(None).mock
     flexmock(module).should_receive('_parse_checks').and_return(flexmock())
     flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_subprocess_mock(
@@ -257,7 +313,7 @@ def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_param
 
 
 def test_check_archives_without_any_checks_should_bail():
-    consistency_config = flexmock()
+    consistency_config = flexmock().should_receive('get').and_return(None).mock
     flexmock(module).should_receive('_parse_checks').and_return(())
     insert_subprocess_never()
 

atticmatic/tests/unit/test_config.py

@@ -197,25 +197,34 @@ def test_parse_section_options_for_missing_section_should_return_empty_dict():
 
 def insert_mock_parser():
     parser = flexmock()
-    parser.should_receive('read')
-    module.ConfigParser = lambda: parser
+    parser.should_receive('read').and_return([flexmock()])
+    module.RawConfigParser = lambda: parser
 
     return parser
 
 
 def test_parse_configuration_should_return_section_configs():
     parser = insert_mock_parser()
+    config_format = (flexmock(name='items'), flexmock(name='things'))
     mock_module = flexmock(module)
     mock_module.should_receive('validate_configuration_format').with_args(
-        parser, module.CONFIG_FORMAT,
+        parser, config_format,
     ).once()
-    mock_section_configs = (flexmock(),) * len(module.CONFIG_FORMAT)
+    mock_section_configs = (flexmock(), flexmock())
 
-    for section_format, section_config in zip(module.CONFIG_FORMAT, mock_section_configs):
+    for section_format, section_config in zip(config_format, mock_section_configs):
         mock_module.should_receive('parse_section_options').with_args(
             parser, section_format,
         ).and_return(section_config).once()
 
-    parsed_config = module.parse_configuration('filename')
+    parsed_config = module.parse_configuration('filename', config_format)
 
-    assert parsed_config == module.Parsed_config(*mock_section_configs)
+    assert parsed_config == type(parsed_config)(*mock_section_configs)
+
+
+def test_parse_configuration_with_file_open_error_should_raise():
+    parser = insert_mock_parser()
+    parser.should_receive('read').and_return([])
+
+    with assert_raises(ValueError):
+        module.parse_configuration('filename', config_format=flexmock())

sample/config

@@ -5,11 +5,20 @@ source_directories: /home /etc
 # Path to local or remote repository.
 repository: user@backupserver:sourcehostname.attic
 
+[storage]
+# Passphrase to unlock the encryption key with. Only use on repositories that
+# were initialized with passphrase/repokey encryption.
+#encryption_passphrase: foo
+# For Borg only, you can specify the type of compression to use when creating
+# archives. See https://borgbackup.github.io/borgbackup/usage.html#borg-create
+# for details. Defaults to no compression.
+#compression: lz4
+
 [retention]
 # Retention policy for how many backups to keep in each category. See
 # https://attic-backup.org/usage.html#attic-prune or
 # https://borgbackup.github.io/borgbackup/usage.html#borg-prune for details.
-#keep_within: 3h
+#keep_within: 3H
 #keep_hourly: 24
 keep_daily: 7
 keep_weekly: 4
@@ -20,5 +29,8 @@ keep_yearly: 1
 [consistency]
 # Space-separated list of consistency checks to run: "repository", "archives",
 # or both. Defaults to both. Set to "disabled" to disable all consistency
-# checks. See https://attic-backup.org/usage.html#attic-check for details.
+# checks. See https://attic-backup.org/usage.html#attic-check or
+# https://borgbackup.github.io/borgbackup/usage.html#borg-check for details.
 checks: repository archives
+# For Borg only, you can restrict the number of checked archives to the last n.
+#check_last: 3

setup.cfg

@@ -1,2 +1,5 @@
+[metadata]
+description-file=README.md
+
 [nosetests]
 detailed-errors=1

setup.py

@@ -1,11 +1,26 @@
 from setuptools import setup, find_packages
 
+
+VERSION = '0.1.7'
+
+
 setup(
     name='atticmatic',
-    version='0.1.1',
+    version=VERSION,
     description='A wrapper script for Attic/Borg backup software that creates and prunes backups',
     author='Dan Helfman',
     author_email='witten@torsion.org',
+    url='https://torsion.org/atticmatic',
+    download_url='https://torsion.org/hg/atticmatic/archive/%s.tar.gz' % VERSION,
+    classifiers=(
+        'Development Status :: 5 - Production/Stable',
+        'Environment :: Console',
+        'Intended Audience :: System Administrators',
+        'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
+        'Programming Language :: Python',
+        'Topic :: Security :: Cryptography',
+        'Topic :: System :: Archiving :: Backup',
+    ),
     packages=find_packages(),
     entry_points={
         'console_scripts': [