New configuration section for customizing which Attic consistency checks run, if any.

.hgtags

@@ -1,2 +1,10 @@
 467d3a3ce9185e84ee51ca9156499162efd94f9a 0.0.2
 7730ae34665c0dedf46deab90b32780abf6dbaff 0.0.3
+4bb2e81fc77038be4499b7ea6797ab7d109460e0 0.0.4
+b31d51b633701554e84f996cc0c73bad2990780b 0.0.5
+b31d51b633701554e84f996cc0c73bad2990780b 0.0.5
+aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5
+aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5
+569aef47a9b25c55b13753f94706f5d330219995 0.0.5
+569aef47a9b25c55b13753f94706f5d330219995 0.0.5
+a03495a8e8b471da63b5e2ae79d3ff9065839c2a 0.0.5

NEWS

@@ -1,3 +1,12 @@
+0.0.6
+
+ * New configuration section for customizing which Attic consistency checks run, if any.
+
+0.0.5
+
+ * Fixed regression with --verbose output being buffered. This means dropping the helpful error
+   message introduced in 0.0.4.
+
 0.0.4
 
  * Now using tox to run tests against multiple versions of Python in one go.

README.md

@@ -26,6 +26,9 @@ Here's an example config file:
     keep_weekly: 4
     keep_monthly: 6
 
+    [consistency]
+    checks: repository archives
+
 Additionally, exclude patterns can be specified in a separate excludes config
 file, one pattern per line.
 
@@ -37,6 +40,18 @@ available](https://torsion.org/hg/atticmatic). It's also mirrored on
 
 ## Setup
 
+To get up and running with Attic, follow the [Attic Quick
+Start](https://attic-backup.org/quickstart.html) guide to create an Attic
+repository on a local or remote host. Note that if you plan to run atticmatic
+on a schedule with cron, and you encrypt your attic repository with a
+passphrase instead of a key file, you'll need to set the `ATTIC_PASSPHRASE`
+environment variable. See [attic's repository encryption
+documentation](https://attic-backup.org/quickstart.html#encrypted-repos) for
+more info.
+
+If the repository is on a remote host, make sure that your local root user has
+key-based ssh access to the desired user account on the remote host.
+
 To install atticmatic, run the following command to download and install it:
 
     sudo pip install --upgrade hg+https://torsion.org/hg/atticmatic
@@ -47,24 +62,7 @@ Then copy the following configuration files:
     sudo mkdir /etc/atticmatic/
     sudo cp sample/config sample/excludes /etc/atticmatic/
 
-Modify those files with your desired configuration, including the path to an
-attic repository.
-
-If you don't yet have an attic repository, then the first time you run
-atticmatic, you'll get an error with information on how to create a repository
-on a local or remote host.
-
-And if the repository is on a remote host, make sure that your local root user
-has key-based ssh access to the desired user account on the remote host.
-
-It is recommended that you create your attic repository with keyfile
-encryption, as passphrase-based encryption is less suited for automated
-backups. If you do plan to run atticmatic on a schedule with cron, and you
-encrypt your attic repository with a passphrase instead of a key file, you'll
-need to set the `ATTIC_PASSPHRASE` environment variable. See [attic's
-repository encryption
-documentation](https://attic-backup.org/quickstart.html#encrypted-repos) for
-more info.
+Lastly, modify those files with your desired configuration.
 
 
 ## Usage

atticmatic/attic.py

@@ -1,10 +1,7 @@
-from __future__ import print_function
 from datetime import datetime
 import os
 import platform
-import re
 import subprocess
-import sys
 
 
 def create_archive(excludes_filename, verbose, source_directories, repository):
@@ -26,17 +23,10 @@ def create_archive(excludes_filename, verbose, source_directories, repository):
         ('--verbose', '--stats') if verbose else ()
     )
 
-    try:
-        subprocess.check_output(command, stderr=subprocess.STDOUT)
-    except subprocess.CalledProcessError as error:
-        print(error.output.strip(), file=sys.stderr)
-
-        if re.search('Error: Repository .* does not exist', error.output):
-            raise RuntimeError('To create a repository, run: attic init --encryption=keyfile {}'.format(repository))
-        raise error
+    subprocess.check_call(command)
 
 
-def make_prune_flags(retention_config):
+def _make_prune_flags(retention_config):
     '''
     Given a retention config dict mapping from option name to value, tranform it into an iterable of
     command-line name-value flag pairs.
@@ -68,22 +58,77 @@ def prune_archives(verbose, repository, retention_config):
         repository,
     ) + tuple(
         element
-        for pair in make_prune_flags(retention_config)
+        for pair in _make_prune_flags(retention_config)
         for element in pair
     ) + (('--verbose',) if verbose else ())
 
     subprocess.check_call(command)
 
 
-def check_archives(verbose, repository):
+DEFAULT_CHECKS = ('repository', 'archives')
+
+
+def _parse_checks(consistency_config):
     '''
-    Given a verbosity flag and a local or remote repository path, check the contained attic archives
-    for consistency.
+    Given a consistency config with a space-separated "checks" option, transform it to a tuple of
+    named checks to run.
+
+    For example, given a retention config of:
+
+        {'checks': 'repository archives'}
+
+    This will be returned as:
+
+        ('repository', 'archives')
+
+    If no "checks" option is present, return the DEFAULT_CHECKS. If the checks value is the string
+    "disabled", return an empty tuple, meaning that no checks should be run.
     '''
+    checks = consistency_config.get('checks', '').strip()
+    if not checks:
+        return DEFAULT_CHECKS
+
+    return tuple(
+        check for check in consistency_config['checks'].split(' ')
+        if check.lower() not in ('disabled', '')
+    )
+
+
+def _make_check_flags(checks):
+    '''
+    Given a parsed sequence of checks, transform it into tuple of command-line flags.
+
+    For example, given parsed checks of:
+
+        ('repository',)
+
+    This will be returned as:
+    
+        ('--repository-only',)
+    '''
+    if checks == DEFAULT_CHECKS:
+        return ()
+
+    return tuple(
+        '--{}-only'.format(check) for check in checks
+    )
+
+
+def check_archives(verbose, repository, consistency_config):
+    '''
+    Given a verbosity flag, a local or remote repository path, and a consistency config dict, check
+    the contained attic archives for consistency.
+
+    If there are no consistency checks to run, skip running them.
+    '''
+    checks = _parse_checks(consistency_config)
+    if not checks:
+        return
+
     command = (
         'attic', 'check',
         repository,
-    ) + (('--verbose',) if verbose else ())
+    ) + _make_check_flags(checks) + (('--verbose',) if verbose else ())
 
     # Attic's check command spews to stdout even without the verbose flag. Suppress it.
     stdout = None if verbose else open(os.devnull, 'w')

atticmatic/command.py

@@ -40,12 +40,12 @@ def parse_arguments(*arguments):
 def main():
     try:
         args = parse_arguments(*sys.argv[1:])
-        location_config, retention_config = parse_configuration(args.config_filename)
-        repository = location_config['repository']
+        config = parse_configuration(args.config_filename)
+        repository = config.location['repository']
 
-        create_archive(args.excludes_filename, args.verbose, **location_config)
-        prune_archives(args.verbose, repository, retention_config)
-        check_archives(args.verbose, repository)
-    except (ValueError, IOError, CalledProcessError, RuntimeError) as error:
+        create_archive(args.excludes_filename, args.verbose, **config.location)
+        prune_archives(args.verbose, repository, config.retention)
+        check_archives(args.verbose, repository, config.consistency)
+    except (ValueError, IOError, CalledProcessError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)

atticmatic/config.py

@@ -39,6 +39,12 @@ CONFIG_FORMAT = (
             option('keep_yearly', int, required=False),
             option('prefix', required=False),
         ),
+    ),
+    Section_format(
+        'consistency',
+        (
+            option('checks', required=False),
+        ),
     )
 )
 
@@ -49,20 +55,34 @@ def validate_configuration_format(parser, config_format):
     configuration file has the expected sections, that any required options are present in those
     sections, and that there aren't any unexpected options.
 
+    A section is required if any of its contained options are required.
+
     Raise ValueError if anything is awry.
     '''
-    section_names = parser.sections()
-    required_section_names = tuple(section.name for section in config_format)
+    section_names = set(parser.sections())
+    required_section_names = tuple(
+        section.name for section in config_format
+        if any(option.required for option in section.options)
+    )
 
-    if set(section_names) != set(required_section_names):
+    unknown_section_names = section_names - set(
+        section_format.name for section_format in config_format
+    )
+    if unknown_section_names:
         raise ValueError(
-            'Expected config sections {} but found sections: {}'.format(
-                ', '.join(required_section_names),
-                ', '.join(section_names)
-            )
+            'Unknown config sections found: {}'.format(', '.join(unknown_section_names))
+        )
+
+    missing_section_names = set(required_section_names) - section_names
+    if missing_section_names:
+        raise ValueError(
+            'Missing config sections: {}'.format(', '.join(missing_section_names))
         )
 
     for section_format in config_format:
+        if section_format.name not in section_names:
+            continue
+
         option_names = parser.options(section_format.name)
         expected_options = section_format.options
 
@@ -90,6 +110,11 @@ def validate_configuration_format(parser, config_format):
             )
 
 
+# Describes a parsed configuration, where each attribute is the name of a configuration file section 
+# and each value is a dict of that section's parsed options.
+Parsed_config = namedtuple('Config', (section_format.name for section_format in CONFIG_FORMAT))
+
+
 def parse_section_options(parser, section_format):
     '''
     Given an open ConfigParser and an expected section format, return the option values from that
@@ -112,8 +137,8 @@ def parse_section_options(parser, section_format):
 
 def parse_configuration(config_filename):
     '''
-    Given a config filename of the expected format, return the parsed configuration as a tuple of
-    (location config, retention config) where each config is a dict of that section's options.
+    Given a config filename of the expected format, return the parsed configuration as Parsed_config
+    data structure.
 
     Raise IOError if the file cannot be read, or ValueError if the format is not as expected.
     '''
@@ -122,7 +147,9 @@ def parse_configuration(config_filename):
 
     validate_configuration_format(parser, CONFIG_FORMAT)
 
-    return tuple(
-        parse_section_options(parser, section_format)
-        for section_format in CONFIG_FORMAT
+    return Parsed_config(
+        *(
+            parse_section_options(parser, section_format)
+            for section_format in CONFIG_FORMAT
+        )
     )

atticmatic/tests/unit/test_attic.py

@@ -1,49 +1,20 @@
 from collections import OrderedDict
-try:
-    # Python 2
-    import __builtin__ as builtins
-except ImportError:
-    # Python 3
-    import builtins
 
 from flexmock import flexmock
-from nose.tools import assert_raises
 
 from atticmatic import attic as module
 
 
-class MockCalledProcessError(Exception):
-    def __init__(self, output):
-        self.output = output
-
-
-def insert_subprocess_check_output_mock(call_command, error_output=None, **kwargs):
-    subprocess = flexmock(CalledProcessError=MockCalledProcessError, STDOUT=flexmock())
-
-    expectation = subprocess.should_receive('check_output').with_args(
-        call_command,
-        stderr=subprocess.STDOUT,
-        **kwargs
-    ).once()
-
-    if error_output:
-        expectation.and_raise(MockCalledProcessError, output=error_output)
-        flexmock(builtins).should_receive('print')
-
-    flexmock(module).subprocess = subprocess
-    return subprocess
-
-
-def insert_subprocess_check_call_mock(call_command, **kwargs):
+def insert_subprocess_mock(check_call_command, **kwargs):
     subprocess = flexmock()
+    subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
+    flexmock(module).subprocess = subprocess
 
-    subprocess.should_receive('check_call').with_args(
-        call_command,
-        **kwargs
-    ).once()
 
+def insert_subprocess_never():
+    subprocess = flexmock()
+    subprocess.should_receive('check_call').never()
     flexmock(module).subprocess = subprocess
-    return subprocess
 
 
 def insert_platform_mock():
@@ -57,7 +28,7 @@ def insert_datetime_mock():
 
 
 def test_create_archive_should_call_attic_with_parameters():
-    insert_subprocess_check_output_mock(
+    insert_subprocess_mock(
         ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar'),
     )
     insert_platform_mock()
@@ -72,7 +43,7 @@ def test_create_archive_should_call_attic_with_parameters():
 
 
 def test_create_archive_with_verbose_should_call_attic_with_verbose_parameters():
-    insert_subprocess_check_output_mock(
+    insert_subprocess_mock(
         (
             'attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar',
             '--verbose', '--stats',
@@ -88,39 +59,6 @@ def test_create_archive_with_verbose_should_call_attic_with_verbose_parameters()
         repository='repo',
     )
 
-def test_create_archive_with_missing_repository_should_raise():
-    insert_subprocess_check_output_mock(
-        ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar'),
-        error_output='Error: Repository repo does not exist',
-    )
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    with assert_raises(RuntimeError):
-        module.create_archive(
-            excludes_filename='excludes',
-            verbose=False,
-            source_directories='foo bar',
-            repository='repo',
-        )
-
-
-def test_create_archive_with_other_error_should_raise():
-    subprocess = insert_subprocess_check_output_mock(
-        ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar'),
-        error_output='Something went wrong',
-    )
-    insert_platform_mock()
-    insert_datetime_mock()
-
-    with assert_raises(subprocess.CalledProcessError):
-        module.create_archive(
-            excludes_filename='excludes',
-            verbose=False,
-            source_directories='foo bar',
-            repository='repo',
-        )
-
 
 BASE_PRUNE_FLAGS = (
     ('--keep-daily', '1'),
@@ -138,17 +76,17 @@ def test_make_prune_flags_should_return_flags_from_config():
         )
     )
 
-    result = module.make_prune_flags(retention_config)
+    result = module._make_prune_flags(retention_config)
 
     assert tuple(result) == BASE_PRUNE_FLAGS
 
 
 def test_prune_archives_should_call_attic_with_parameters():
     retention_config = flexmock()
-    flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return(
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS,
     )
-    insert_subprocess_check_call_mock(
+    insert_subprocess_mock(
         (
             'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly',
             '3',
@@ -164,10 +102,10 @@ def test_prune_archives_should_call_attic_with_parameters():
 
 def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters():
     retention_config = flexmock()
-    flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return(
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS,
     )
-    insert_subprocess_check_call_mock(
+    insert_subprocess_mock(
         (
             'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly',
             '3', '--verbose',
@@ -181,9 +119,48 @@ def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters()
     )
 
 
+def test_parse_checks_returns_them_as_tuple():
+    checks = module._parse_checks({'checks': 'foo disabled bar'})
+
+    assert checks == ('foo', 'bar')
+
+
+def test_parse_checks_with_missing_value_returns_defaults():
+    checks = module._parse_checks({})
+
+    assert checks == module.DEFAULT_CHECKS
+
+
+def test_parse_checks_with_blank_value_returns_defaults():
+    checks = module._parse_checks({'checks': ''})
+
+    assert checks == module.DEFAULT_CHECKS
+
+
+def test_parse_checks_with_disabled_returns_no_checks():
+    checks = module._parse_checks({'checks': 'disabled'})
+
+    assert checks == ()
+
+
+def test_make_check_flags_with_checks_returns_flags():
+    flags = module._make_check_flags(('foo', 'bar'))
+
+    assert flags == ('--foo-only', '--bar-only')
+
+
+def test_make_check_flags_with_default_checks_returns_no_flags():
+    flags = module._make_check_flags(module.DEFAULT_CHECKS)
+
+    assert flags == ()
+
+
 def test_check_archives_should_call_attic_with_parameters():
+    consistency_config = flexmock()
+    flexmock(module).should_receive('_parse_checks').and_return(flexmock())
+    flexmock(module).should_receive('_make_check_flags').and_return(())
     stdout = flexmock()
-    insert_subprocess_check_call_mock(
+    insert_subprocess_mock(
         ('attic', 'check', 'repo'),
         stdout=stdout,
     )
@@ -195,11 +172,15 @@ def test_check_archives_should_call_attic_with_parameters():
     module.check_archives(
         verbose=False,
         repository='repo',
+        consistency_config=consistency_config,
     )
 
 
 def test_check_archives_with_verbose_should_call_attic_with_verbose_parameters():
-    insert_subprocess_check_call_mock(
+    consistency_config = flexmock()
+    flexmock(module).should_receive('_parse_checks').and_return(flexmock())
+    flexmock(module).should_receive('_make_check_flags').and_return(())
+    insert_subprocess_mock(
         ('attic', 'check', 'repo', '--verbose'),
         stdout=None,
     )
@@ -209,4 +190,19 @@ def test_check_archives_with_verbose_should_call_attic_with_verbose_parameters()
     module.check_archives(
         verbose=True,
         repository='repo',
+        consistency_config=consistency_config,
     )
+
+
+def test_check_archives_without_any_checks_should_bail():
+    consistency_config = flexmock()
+    flexmock(module).should_receive('_parse_checks').and_return(())
+    insert_subprocess_never()
+
+    module.check_archives(
+        verbose=False,
+        repository='repo',
+        consistency_config=consistency_config,
+    )
+
+

atticmatic/tests/unit/test_config.py

@@ -41,19 +41,55 @@ def test_validate_configuration_format_with_valid_config_should_not_raise():
     module.validate_configuration_format(parser, config_format)
 
 
-def test_validate_configuration_format_with_missing_section_should_raise():
+def test_validate_configuration_format_with_missing_required_section_should_raise():
     parser = flexmock()
     parser.should_receive('sections').and_return(('section',))
     config_format = (
-        module.Section_format('section', options=()),
-        module.Section_format('missing', options=()),
+        module.Section_format(
+            'section',
+            options=(
+                module.Config_option('stuff', str, required=True),
+            ),
+        ),
+        # At least one option in this section is required, so the section is required.
+        module.Section_format(
+            'missing',
+            options=(
+                module.Config_option('such', str, required=False),
+                module.Config_option('things', str, required=True),
+            ),
+        ),
     )
 
     with assert_raises(ValueError):
         module.validate_configuration_format(parser, config_format)
 
 
-def test_validate_configuration_format_with_extra_section_should_raise():
+def test_validate_configuration_format_with_missing_optional_section_should_not_raise():
+    parser = flexmock()
+    parser.should_receive('sections').and_return(('section',))
+    parser.should_receive('options').with_args('section').and_return(('stuff',))
+    config_format = (
+        module.Section_format(
+            'section',
+            options=(
+                module.Config_option('stuff', str, required=True),
+            ),
+        ),
+        # No options in the section are required, so the section is optional.
+        module.Section_format(
+            'missing',
+            options=(
+                module.Config_option('such', str, required=False),
+                module.Config_option('things', str, required=False),
+            ),
+        ),
+    )
+
+    module.validate_configuration_format(parser, config_format)
+
+
+def test_validate_configuration_format_with_unknown_section_should_raise():
     parser = flexmock()
     parser.should_receive('sections').and_return(('section', 'extra'))
     config_format = (
@@ -139,6 +175,26 @@ def test_parse_section_options_should_return_section_options():
     )
 
 
+def test_parse_section_options_for_missing_section_should_return_empty_dict():
+    parser = flexmock()
+    parser.should_receive('get').never()
+    parser.should_receive('getint').never()
+    parser.should_receive('has_option').with_args('section', 'foo').and_return(False)
+    parser.should_receive('has_option').with_args('section', 'bar').and_return(False)
+
+    section_format = module.Section_format(
+        'section',
+        (
+            module.Config_option('foo', str, required=False),
+            module.Config_option('bar', int, required=False),
+        ),
+    )
+
+    config = module.parse_section_options(parser, section_format)
+
+    assert config == OrderedDict()
+
+
 def insert_mock_parser():
     parser = flexmock()
     parser.should_receive('readfp')
@@ -154,13 +210,13 @@ def test_parse_configuration_should_return_section_configs():
     mock_module.should_receive('validate_configuration_format').with_args(
         parser, module.CONFIG_FORMAT,
     ).once()
-    mock_section_configs = (flexmock(), flexmock())
+    mock_section_configs = (flexmock(),) * len(module.CONFIG_FORMAT)
 
     for section_format, section_config in zip(module.CONFIG_FORMAT, mock_section_configs):
         mock_module.should_receive('parse_section_options').with_args(
             parser, section_format,
         ).and_return(section_config).once()
 
-    section_configs = module.parse_configuration('filename')
+    parsed_config = module.parse_configuration('filename')
 
-    assert section_configs == mock_section_configs
+    assert parsed_config == module.Parsed_config(*mock_section_configs)

sample/config

@@ -6,8 +6,8 @@ source_directories: /home /etc
 repository: user@backupserver:sourcehostname.attic
 
 [retention]
-# Retention policy for how many backups to keep in each category.
-# See https://attic-backup.org/usage.html#attic-prune for details.
+# Retention policy for how many backups to keep in each category. See
+# https://attic-backup.org/usage.html#attic-prune for details.
 #keep_within: 3h
 #keep_hourly: 24
 keep_daily: 7
@@ -15,3 +15,9 @@ keep_weekly: 4
 keep_monthly: 6
 keep_yearly: 1
 #prefix: sourcehostname
+
+[consistency]
+# Space-separated list of consistency checks to run: "repository", "archives",
+# or both. Defaults to both. Set to "disabled" to disable all consistency
+# checks. See https://attic-backup.org/usage.html#attic-check for details.
+checks: repository archives

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='atticmatic',
-    version='0.0.4',
+    version='0.0.6',
     description='A wrapper script for Attic backup software that creates and prunes backups',
     author='Dan Helfman',
     author_email='witten@torsion.org',

tox.ini

@@ -5,4 +5,4 @@ skipsdist=True
 [testenv]
 usedevelop=True
 deps=-rtest_requirements.txt
-commands = nosetests
+commands = nosetests []