New configuration section for customizing which Attic consistency checks run, if any.

.hgignore

@@ -2,3 +2,4 @@ syntax: glob
 *.egg-info
 *.pyc
 *.swp
+.tox

.hgtags

@@ -1 +1,10 @@
 467d3a3ce9185e84ee51ca9156499162efd94f9a 0.0.2
+7730ae34665c0dedf46deab90b32780abf6dbaff 0.0.3
+4bb2e81fc77038be4499b7ea6797ab7d109460e0 0.0.4
+b31d51b633701554e84f996cc0c73bad2990780b 0.0.5
+b31d51b633701554e84f996cc0c73bad2990780b 0.0.5
+aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5
+aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5
+569aef47a9b25c55b13753f94706f5d330219995 0.0.5
+569aef47a9b25c55b13753f94706f5d330219995 0.0.5
+a03495a8e8b471da63b5e2ae79d3ff9065839c2a 0.0.5

NEWS

@@ -1,3 +1,19 @@
+0.0.6
+
+ * New configuration section for customizing which Attic consistency checks run, if any.
+
+0.0.5
+
+ * Fixed regression with --verbose output being buffered. This means dropping the helpful error
+   message introduced in 0.0.4.
+
+0.0.4
+
+ * Now using tox to run tests against multiple versions of Python in one go.
+ * Helpful error message about how to create a repository if one is missing.
+ * Troubleshooting section with steps to deal with broken pipes.
+ * Nosetests config file (setup.cfg) with defaults.
+
 0.0.3
 
  * After pruning, run attic's consistency checks on all archives.

README.md

@@ -26,6 +26,9 @@ Here's an example config file:
     keep_weekly: 4
     keep_monthly: 6
 
+    [consistency]
+    checks: repository archives
+
 Additionally, exclude patterns can be specified in a separate excludes config
 file, one pattern per line.
 
@@ -51,7 +54,7 @@ key-based ssh access to the desired user account on the remote host.
 
 To install atticmatic, run the following command to download and install it:
 
-    sudo pip install hg+https://torsion.org/hg/atticmatic
+    sudo pip install --upgrade hg+https://torsion.org/hg/atticmatic
 
 Then copy the following configuration files:
 
@@ -85,13 +88,34 @@ If you'd like to see the available command-line arguments, view the help:
 
 ## Running tests
 
-To install test-specific dependencies, first run:
+First install tox, which is used for setting up testing environments:
 
-    sudo python setup.py test
+    pip install tox
 
-To actually run tests, run:
+Then, to actually run tests, run:
 
-    nosetests --detailed-errors
+    tox
+
+
+## Troubleshooting
+
+### Broken pipe with remote repository
+
+When running atticmatic on a large remote repository, you may receive errors
+like the following, particularly while "attic check" is valiating backups for
+consistency:
+
+    Write failed: Broken pipe
+    attic: Error: Connection closed by remote host
+
+This error can be caused by an ssh timeout, which you can rectify by adding
+the following to the ~/.ssh/config file on the client:
+
+    Host *
+        ServerAliveInterval 120
+
+This should make the client keep the connection alive while validating
+backups.
 
 
 ## Feedback

atticmatic/attic.py

@@ -26,7 +26,7 @@ def create_archive(excludes_filename, verbose, source_directories, repository):
     subprocess.check_call(command)
 
 
-def make_prune_flags(retention_config):
+def _make_prune_flags(retention_config):
     '''
     Given a retention config dict mapping from option name to value, tranform it into an iterable of
     command-line name-value flag pairs.
@@ -58,22 +58,77 @@ def prune_archives(verbose, repository, retention_config):
         repository,
     ) + tuple(
         element
-        for pair in make_prune_flags(retention_config)
+        for pair in _make_prune_flags(retention_config)
         for element in pair
     ) + (('--verbose',) if verbose else ())
 
     subprocess.check_call(command)
 
 
-def check_archives(verbose, repository):
+DEFAULT_CHECKS = ('repository', 'archives')
+
+
+def _parse_checks(consistency_config):
     '''
-    Given a verbosity flag and a local or remote repository path, check the contained attic archives
-    for consistency.
+    Given a consistency config with a space-separated "checks" option, transform it to a tuple of
+    named checks to run.
+
+    For example, given a retention config of:
+
+        {'checks': 'repository archives'}
+
+    This will be returned as:
+
+        ('repository', 'archives')
+
+    If no "checks" option is present, return the DEFAULT_CHECKS. If the checks value is the string
+    "disabled", return an empty tuple, meaning that no checks should be run.
     '''
+    checks = consistency_config.get('checks', '').strip()
+    if not checks:
+        return DEFAULT_CHECKS
+
+    return tuple(
+        check for check in consistency_config['checks'].split(' ')
+        if check.lower() not in ('disabled', '')
+    )
+
+
+def _make_check_flags(checks):
+    '''
+    Given a parsed sequence of checks, transform it into tuple of command-line flags.
+
+    For example, given parsed checks of:
+
+        ('repository',)
+
+    This will be returned as:
+    
+        ('--repository-only',)
+    '''
+    if checks == DEFAULT_CHECKS:
+        return ()
+
+    return tuple(
+        '--{}-only'.format(check) for check in checks
+    )
+
+
+def check_archives(verbose, repository, consistency_config):
+    '''
+    Given a verbosity flag, a local or remote repository path, and a consistency config dict, check
+    the contained attic archives for consistency.
+
+    If there are no consistency checks to run, skip running them.
+    '''
+    checks = _parse_checks(consistency_config)
+    if not checks:
+        return
+
     command = (
         'attic', 'check',
         repository,
-    ) + (('--verbose',) if verbose else ())
+    ) + _make_check_flags(checks) + (('--verbose',) if verbose else ())
 
     # Attic's check command spews to stdout even without the verbose flag. Suppress it.
     stdout = None if verbose else open(os.devnull, 'w')

atticmatic/command.py

@@ -40,12 +40,12 @@ def parse_arguments(*arguments):
 def main():
     try:
         args = parse_arguments(*sys.argv[1:])
-        location_config, retention_config = parse_configuration(args.config_filename)
-        repository = location_config['repository']
+        config = parse_configuration(args.config_filename)
+        repository = config.location['repository']
 
-        create_archive(args.excludes_filename, args.verbose, **location_config)
-        prune_archives(args.verbose, repository, retention_config)
-        check_archives(args.verbose, repository)
+        create_archive(args.excludes_filename, args.verbose, **config.location)
+        prune_archives(args.verbose, repository, config.retention)
+        check_archives(args.verbose, repository, config.consistency)
     except (ValueError, IOError, CalledProcessError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)

atticmatic/config.py

@@ -39,6 +39,12 @@ CONFIG_FORMAT = (
             option('keep_yearly', int, required=False),
             option('prefix', required=False),
         ),
+    ),
+    Section_format(
+        'consistency',
+        (
+            option('checks', required=False),
+        ),
     )
 )
 
@@ -49,20 +55,34 @@ def validate_configuration_format(parser, config_format):
     configuration file has the expected sections, that any required options are present in those
     sections, and that there aren't any unexpected options.
 
+    A section is required if any of its contained options are required.
+
     Raise ValueError if anything is awry.
     '''
-    section_names = parser.sections()
-    required_section_names = tuple(section.name for section in config_format)
+    section_names = set(parser.sections())
+    required_section_names = tuple(
+        section.name for section in config_format
+        if any(option.required for option in section.options)
+    )
 
-    if set(section_names) != set(required_section_names):
+    unknown_section_names = section_names - set(
+        section_format.name for section_format in config_format
+    )
+    if unknown_section_names:
         raise ValueError(
-            'Expected config sections {} but found sections: {}'.format(
-                ', '.join(required_section_names),
-                ', '.join(section_names)
-            )
+            'Unknown config sections found: {}'.format(', '.join(unknown_section_names))
+        )
+
+    missing_section_names = set(required_section_names) - section_names
+    if missing_section_names:
+        raise ValueError(
+            'Missing config sections: {}'.format(', '.join(missing_section_names))
         )
 
     for section_format in config_format:
+        if section_format.name not in section_names:
+            continue
+
         option_names = parser.options(section_format.name)
         expected_options = section_format.options
 
@@ -90,6 +110,11 @@ def validate_configuration_format(parser, config_format):
             )
 
 
+# Describes a parsed configuration, where each attribute is the name of a configuration file section 
+# and each value is a dict of that section's parsed options.
+Parsed_config = namedtuple('Config', (section_format.name for section_format in CONFIG_FORMAT))
+
+
 def parse_section_options(parser, section_format):
     '''
     Given an open ConfigParser and an expected section format, return the option values from that
@@ -112,8 +137,8 @@ def parse_section_options(parser, section_format):
 
 def parse_configuration(config_filename):
     '''
-    Given a config filename of the expected format, return the parsed configuration as a tuple of
-    (location config, retention config) where each config is a dict of that section's options.
+    Given a config filename of the expected format, return the parsed configuration as Parsed_config
+    data structure.
 
     Raise IOError if the file cannot be read, or ValueError if the format is not as expected.
     '''
@@ -122,7 +147,9 @@ def parse_configuration(config_filename):
 
     validate_configuration_format(parser, CONFIG_FORMAT)
 
-    return tuple(
-        parse_section_options(parser, section_format)
-        for section_format in CONFIG_FORMAT
+    return Parsed_config(
+        *(
+            parse_section_options(parser, section_format)
+            for section_format in CONFIG_FORMAT
+        )
     )

atticmatic/tests/unit/test_attic.py

@@ -11,6 +11,12 @@ def insert_subprocess_mock(check_call_command, **kwargs):
     flexmock(module).subprocess = subprocess
 
 
+def insert_subprocess_never():
+    subprocess = flexmock()
+    subprocess.should_receive('check_call').never()
+    flexmock(module).subprocess = subprocess
+
+
 def insert_platform_mock():
     flexmock(module).platform = flexmock().should_receive('node').and_return('host').mock
 
@@ -70,14 +76,14 @@ def test_make_prune_flags_should_return_flags_from_config():
         )
     )
 
-    result = module.make_prune_flags(retention_config)
+    result = module._make_prune_flags(retention_config)
 
     assert tuple(result) == BASE_PRUNE_FLAGS
 
 
 def test_prune_archives_should_call_attic_with_parameters():
     retention_config = flexmock()
-    flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return(
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS,
     )
     insert_subprocess_mock(
@@ -96,7 +102,7 @@ def test_prune_archives_should_call_attic_with_parameters():
 
 def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters():
     retention_config = flexmock()
-    flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return(
+    flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS,
     )
     insert_subprocess_mock(
@@ -113,7 +119,46 @@ def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters()
     )
 
 
+def test_parse_checks_returns_them_as_tuple():
+    checks = module._parse_checks({'checks': 'foo disabled bar'})
+
+    assert checks == ('foo', 'bar')
+
+
+def test_parse_checks_with_missing_value_returns_defaults():
+    checks = module._parse_checks({})
+
+    assert checks == module.DEFAULT_CHECKS
+
+
+def test_parse_checks_with_blank_value_returns_defaults():
+    checks = module._parse_checks({'checks': ''})
+
+    assert checks == module.DEFAULT_CHECKS
+
+
+def test_parse_checks_with_disabled_returns_no_checks():
+    checks = module._parse_checks({'checks': 'disabled'})
+
+    assert checks == ()
+
+
+def test_make_check_flags_with_checks_returns_flags():
+    flags = module._make_check_flags(('foo', 'bar'))
+
+    assert flags == ('--foo-only', '--bar-only')
+
+
+def test_make_check_flags_with_default_checks_returns_no_flags():
+    flags = module._make_check_flags(module.DEFAULT_CHECKS)
+
+    assert flags == ()
+
+
 def test_check_archives_should_call_attic_with_parameters():
+    consistency_config = flexmock()
+    flexmock(module).should_receive('_parse_checks').and_return(flexmock())
+    flexmock(module).should_receive('_make_check_flags').and_return(())
     stdout = flexmock()
     insert_subprocess_mock(
         ('attic', 'check', 'repo'),
@@ -127,10 +172,14 @@ def test_check_archives_should_call_attic_with_parameters():
     module.check_archives(
         verbose=False,
         repository='repo',
+        consistency_config=consistency_config,
     )
 
 
 def test_check_archives_with_verbose_should_call_attic_with_verbose_parameters():
+    consistency_config = flexmock()
+    flexmock(module).should_receive('_parse_checks').and_return(flexmock())
+    flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_subprocess_mock(
         ('attic', 'check', 'repo', '--verbose'),
         stdout=None,
@@ -141,4 +190,19 @@ def test_check_archives_with_verbose_should_call_attic_with_verbose_parameters()
     module.check_archives(
         verbose=True,
         repository='repo',
+        consistency_config=consistency_config,
     )
+
+
+def test_check_archives_without_any_checks_should_bail():
+    consistency_config = flexmock()
+    flexmock(module).should_receive('_parse_checks').and_return(())
+    insert_subprocess_never()
+
+    module.check_archives(
+        verbose=False,
+        repository='repo',
+        consistency_config=consistency_config,
+    )
+
+

atticmatic/tests/unit/test_config.py

@@ -41,19 +41,55 @@ def test_validate_configuration_format_with_valid_config_should_not_raise():
     module.validate_configuration_format(parser, config_format)
 
 
-def test_validate_configuration_format_with_missing_section_should_raise():
+def test_validate_configuration_format_with_missing_required_section_should_raise():
     parser = flexmock()
     parser.should_receive('sections').and_return(('section',))
     config_format = (
-        module.Section_format('section', options=()),
-        module.Section_format('missing', options=()),
+        module.Section_format(
+            'section',
+            options=(
+                module.Config_option('stuff', str, required=True),
+            ),
+        ),
+        # At least one option in this section is required, so the section is required.
+        module.Section_format(
+            'missing',
+            options=(
+                module.Config_option('such', str, required=False),
+                module.Config_option('things', str, required=True),
+            ),
+        ),
     )
 
     with assert_raises(ValueError):
         module.validate_configuration_format(parser, config_format)
 
 
-def test_validate_configuration_format_with_extra_section_should_raise():
+def test_validate_configuration_format_with_missing_optional_section_should_not_raise():
+    parser = flexmock()
+    parser.should_receive('sections').and_return(('section',))
+    parser.should_receive('options').with_args('section').and_return(('stuff',))
+    config_format = (
+        module.Section_format(
+            'section',
+            options=(
+                module.Config_option('stuff', str, required=True),
+            ),
+        ),
+        # No options in the section are required, so the section is optional.
+        module.Section_format(
+            'missing',
+            options=(
+                module.Config_option('such', str, required=False),
+                module.Config_option('things', str, required=False),
+            ),
+        ),
+    )
+
+    module.validate_configuration_format(parser, config_format)
+
+
+def test_validate_configuration_format_with_unknown_section_should_raise():
     parser = flexmock()
     parser.should_receive('sections').and_return(('section', 'extra'))
     config_format = (
@@ -139,6 +175,26 @@ def test_parse_section_options_should_return_section_options():
     )
 
 
+def test_parse_section_options_for_missing_section_should_return_empty_dict():
+    parser = flexmock()
+    parser.should_receive('get').never()
+    parser.should_receive('getint').never()
+    parser.should_receive('has_option').with_args('section', 'foo').and_return(False)
+    parser.should_receive('has_option').with_args('section', 'bar').and_return(False)
+
+    section_format = module.Section_format(
+        'section',
+        (
+            module.Config_option('foo', str, required=False),
+            module.Config_option('bar', int, required=False),
+        ),
+    )
+
+    config = module.parse_section_options(parser, section_format)
+
+    assert config == OrderedDict()
+
+
 def insert_mock_parser():
     parser = flexmock()
     parser.should_receive('readfp')
@@ -154,13 +210,13 @@ def test_parse_configuration_should_return_section_configs():
     mock_module.should_receive('validate_configuration_format').with_args(
         parser, module.CONFIG_FORMAT,
     ).once()
-    mock_section_configs = (flexmock(), flexmock())
+    mock_section_configs = (flexmock(),) * len(module.CONFIG_FORMAT)
 
     for section_format, section_config in zip(module.CONFIG_FORMAT, mock_section_configs):
         mock_module.should_receive('parse_section_options').with_args(
             parser, section_format,
         ).and_return(section_config).once()
 
-    section_configs = module.parse_configuration('filename')
+    parsed_config = module.parse_configuration('filename')
 
-    assert section_configs == mock_section_configs
+    assert parsed_config == module.Parsed_config(*mock_section_configs)

sample/config

@@ -6,8 +6,8 @@ source_directories: /home /etc
 repository: user@backupserver:sourcehostname.attic
 
 [retention]
-# Retention policy for how many backups to keep in each category.
-# See https://attic-backup.org/usage.html#attic-prune for details.
+# Retention policy for how many backups to keep in each category. See
+# https://attic-backup.org/usage.html#attic-prune for details.
 #keep_within: 3h
 #keep_hourly: 24
 keep_daily: 7
@@ -15,3 +15,9 @@ keep_weekly: 4
 keep_monthly: 6
 keep_yearly: 1
 #prefix: sourcehostname
+
+[consistency]
+# Space-separated list of consistency checks to run: "repository", "archives",
+# or both. Defaults to both. Set to "disabled" to disable all consistency
+# checks. See https://attic-backup.org/usage.html#attic-check for details.
+checks: repository archives

setup.cfg

@@ -0,0 +1,2 @@
+[nosetests]
+detailed-errors=1

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='atticmatic',
-    version='0.0.2',
+    version='0.0.6',
     description='A wrapper script for Attic backup software that creates and prunes backups',
     author='Dan Helfman',
     author_email='witten@torsion.org',

test_requirements.txt

@@ -0,0 +1,2 @@
+flexmock==0.9.7
+nose==1.3.4

tox.ini

@@ -0,0 +1,8 @@
+[tox]
+envlist=py27,py34
+skipsdist=True
+
+[testenv]
+usedevelop=True
+deps=-rtest_requirements.txt
+commands = nosetests []