Releasing 0.0.4.

.hgignore

@@ -2,3 +2,4 @@ syntax: glob
 *.egg-info
 *.pyc
 *.swp
+.tox

.hgtags

@@ -0,0 +1,2 @@
+467d3a3ce9185e84ee51ca9156499162efd94f9a 0.0.2
+7730ae34665c0dedf46deab90b32780abf6dbaff 0.0.3

NEWS

@@ -1,3 +1,16 @@
+0.0.4
+
+ * Now using tox to run tests against multiple versions of Python in one go.
+ * Helpful error message about how to create a repository if one is missing.
+ * Troubleshooting section with steps to deal with broken pipes.
+ * Nosetests config file (setup.cfg) with defaults.
+
+0.0.3
+
+ * After pruning, run attic's consistency checks on all archives.
+ * Integration tests for argument parsing.
+ * Documentation updates about repository encryption.
+
 0.0.2
 
  * Configuration support for additional attic prune flags: keep_within, keep_hourly, keep_yearly,

README.md

@@ -5,10 +5,11 @@ save_as: atticmatic/index.html
 ## Overview
 
 atticmatic is a simple Python wrapper script for the [Attic backup
-software](https://attic-backup.org/) that initiates a backup and prunes any
-old backups according to a retention policy. The script supports specifying
-your settings in a declarative configuration file rather than having to put
-them all on the command-line, and handles common errors.
+software](https://attic-backup.org/) that initiates a backup, prunes any old
+backups according to a retention policy, and validates backups for
+consistency. The script supports specifying your settings in a declarative
+configuration file rather than having to put them all on the command-line, and
+handles common errors.
 
 Here's an example config file:
 
@@ -36,16 +37,9 @@ available](https://torsion.org/hg/atticmatic). It's also mirrored on
 
 ## Setup
 
-To get up and running with Attic, follow the [Attic Quick
-Start](https://attic-backup.org/quickstart.html) guide to create an Attic
-repository on a local or remote host.
-
-If the repository is on a remote host, make sure that your local root user has
-key-based ssh access to the desired user account on the remote host.
-
 To install atticmatic, run the following command to download and install it:
 
-    sudo pip install hg+https://torsion.org/hg/atticmatic
+    sudo pip install --upgrade hg+https://torsion.org/hg/atticmatic
 
 Then copy the following configuration files:
 
@@ -53,7 +47,24 @@ Then copy the following configuration files:
     sudo mkdir /etc/atticmatic/
     sudo cp sample/config sample/excludes /etc/atticmatic/
 
-Lastly, modify those files with your desired configuration.
+Modify those files with your desired configuration, including the path to an
+attic repository.
+
+If you don't yet have an attic repository, then the first time you run
+atticmatic, you'll get an error with information on how to create a repository
+on a local or remote host.
+
+And if the repository is on a remote host, make sure that your local root user
+has key-based ssh access to the desired user account on the remote host.
+
+It is recommended that you create your attic repository with keyfile
+encryption, as passphrase-based encryption is less suited for automated
+backups. If you do plan to run atticmatic on a schedule with cron, and you
+encrypt your attic repository with a passphrase instead of a key file, you'll
+need to set the `ATTIC_PASSPHRASE` environment variable. See [attic's
+repository encryption
+documentation](https://attic-backup.org/quickstart.html#encrypted-repos) for
+more info.
 
 
 ## Usage
@@ -63,7 +74,9 @@ arguments:
 
     atticmatic
 
-This will also prune any old backups as per the configured retention policy.
+This will also prune any old backups as per the configured retention policy,
+and check backups for consistency problems due to things like file damage.
+
 By default, the backup will proceed silently except in the case of errors. But
 if you'd like to to get additional information about the progress of the
 backup as it proceeds, use the verbose option instead:
@@ -77,13 +90,34 @@ If you'd like to see the available command-line arguments, view the help:
 
 ## Running tests
 
-To install test-specific dependencies, first run:
+First install tox, which is used for setting up testing environments:
 
-    sudo python setup.py test
+    pip install tox
 
-To actually run tests, run:
+Then, to actually run tests, run:
 
-    nosetests --detailed-errors
+    tox
+
+
+## Troubleshooting
+
+### Broken pipe with remote repository
+
+When running atticmatic on a large remote repository, you may receive errors
+like the following, particularly while "attic check" is valiating backups for
+consistency:
+
+    Write failed: Broken pipe
+    attic: Error: Connection closed by remote host
+
+This error can be caused by an ssh timeout, which you can rectify by adding
+the following to the ~/.ssh/config file on the client:
+
+    Host *
+        ServerAliveInterval 120
+
+This should make the client keep the connection alive while validating
+backups.
 
 
 ## Feedback

atticmatic/attic.py

@@ -1,7 +1,10 @@
+from __future__ import print_function
 from datetime import datetime
-
+import os
 import platform
+import re
 import subprocess
+import sys
 
 
 def create_archive(excludes_filename, verbose, source_directories, repository):
@@ -23,7 +26,14 @@ def create_archive(excludes_filename, verbose, source_directories, repository):
         ('--verbose', '--stats') if verbose else ()
     )
 
-    subprocess.check_call(command)
+    try:
+        subprocess.check_output(command, stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError as error:
+        print(error.output.strip(), file=sys.stderr)
+
+        if re.search('Error: Repository .* does not exist', error.output):
+            raise RuntimeError('To create a repository, run: attic init --encryption=keyfile {}'.format(repository))
+        raise error
 
 
 def make_prune_flags(retention_config):
@@ -63,3 +73,19 @@ def prune_archives(verbose, repository, retention_config):
     ) + (('--verbose',) if verbose else ())
 
     subprocess.check_call(command)
+
+
+def check_archives(verbose, repository):
+    '''
+    Given a verbosity flag and a local or remote repository path, check the contained attic archives
+    for consistency.
+    '''
+    command = (
+        'attic', 'check',
+        repository,
+    ) + (('--verbose',) if verbose else ())
+
+    # Attic's check command spews to stdout even without the verbose flag. Suppress it.
+    stdout = None if verbose else open(os.devnull, 'w')
+
+    subprocess.check_call(command, stdout=stdout)

atticmatic/command.py

@@ -3,25 +3,29 @@ from argparse import ArgumentParser
 from subprocess import CalledProcessError
 import sys
 
-from atticmatic.attic import create_archive, prune_archives
+from atticmatic.attic import check_archives, create_archive, prune_archives
 from atticmatic.config import parse_configuration
 
 
-def parse_arguments():
+DEFAULT_CONFIG_FILENAME = '/etc/atticmatic/config'
+DEFAULT_EXCLUDES_FILENAME = '/etc/atticmatic/excludes'
+
+
+def parse_arguments(*arguments):
     '''
-    Parse the command-line arguments from sys.argv and return them as an ArgumentParser instance.
+    Parse the given command-line arguments and return them as an ArgumentParser instance.
     '''
     parser = ArgumentParser()
     parser.add_argument(
         '-c', '--config',
         dest='config_filename',
-        default='/etc/atticmatic/config',
+        default=DEFAULT_CONFIG_FILENAME,
         help='Configuration filename',
     )
     parser.add_argument(
         '--excludes',
         dest='excludes_filename',
-        default='/etc/atticmatic/excludes',
+        default=DEFAULT_EXCLUDES_FILENAME,
         help='Excludes filename',
     )
     parser.add_argument(
@@ -30,16 +34,18 @@ def parse_arguments():
         help='Display verbose progress information',
     )
 
-    return parser.parse_args()
+    return parser.parse_args(arguments)
 
 
 def main():
     try:
-        args = parse_arguments()
+        args = parse_arguments(*sys.argv[1:])
         location_config, retention_config = parse_configuration(args.config_filename)
+        repository = location_config['repository']
 
         create_archive(args.excludes_filename, args.verbose, **location_config)
-        prune_archives(args.verbose, location_config['repository'], retention_config)
-    except (ValueError, IOError, CalledProcessError) as error:
+        prune_archives(args.verbose, repository, retention_config)
+        check_archives(args.verbose, repository)
+    except (ValueError, IOError, CalledProcessError, RuntimeError) as error:
         print(error, file=sys.stderr)
         sys.exit(1)

atticmatic/tests/integration/test_command.py

@@ -0,0 +1,40 @@
+import sys
+
+from nose.tools import assert_raises
+
+from atticmatic import command as module
+
+
+def test_parse_arguments_with_no_arguments_uses_defaults():
+    parser = module.parse_arguments()
+
+    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
+    assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME
+    assert parser.verbose == False
+
+
+def test_parse_arguments_with_filename_arguments_overrides_defaults():
+    parser = module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes')
+
+    assert parser.config_filename == 'myconfig'
+    assert parser.excludes_filename == 'myexcludes'
+    assert parser.verbose == False
+
+
+def test_parse_arguments_with_verbose_flag_overrides_default():
+    parser = module.parse_arguments('--verbose')
+
+    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
+    assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME
+    assert parser.verbose == True
+
+
+def test_parse_arguments_with_invalid_arguments_exits():
+    original_stderr = sys.stderr
+    sys.stderr = sys.stdout
+
+    try:
+        with assert_raises(SystemExit):
+            module.parse_arguments('--posix-me-harder')
+    finally:
+        sys.stderr = original_stderr

atticmatic/tests/unit/test_attic.py

@@ -1,14 +1,49 @@
 from collections import OrderedDict
+try:
+    # Python 2
+    import __builtin__ as builtins
+except ImportError:
+    # Python 3
+    import builtins
 
 from flexmock import flexmock
+from nose.tools import assert_raises
 
 from atticmatic import attic as module
 
 
-def insert_subprocess_mock(check_call_command):
-    subprocess = flexmock()
-    subprocess.should_receive('check_call').with_args(check_call_command).once()
+class MockCalledProcessError(Exception):
+    def __init__(self, output):
+        self.output = output
+
+
+def insert_subprocess_check_output_mock(call_command, error_output=None, **kwargs):
+    subprocess = flexmock(CalledProcessError=MockCalledProcessError, STDOUT=flexmock())
+
+    expectation = subprocess.should_receive('check_output').with_args(
+        call_command,
+        stderr=subprocess.STDOUT,
+        **kwargs
+    ).once()
+
+    if error_output:
+        expectation.and_raise(MockCalledProcessError, output=error_output)
+        flexmock(builtins).should_receive('print')
+
     flexmock(module).subprocess = subprocess
+    return subprocess
+
+
+def insert_subprocess_check_call_mock(call_command, **kwargs):
+    subprocess = flexmock()
+
+    subprocess.should_receive('check_call').with_args(
+        call_command,
+        **kwargs
+    ).once()
+
+    flexmock(module).subprocess = subprocess
+    return subprocess
 
 
 def insert_platform_mock():
@@ -22,7 +57,7 @@ def insert_datetime_mock():
 
 
 def test_create_archive_should_call_attic_with_parameters():
-    insert_subprocess_mock(
+    insert_subprocess_check_output_mock(
         ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar'),
     )
     insert_platform_mock()
@@ -37,7 +72,7 @@ def test_create_archive_should_call_attic_with_parameters():
 
 
 def test_create_archive_with_verbose_should_call_attic_with_verbose_parameters():
-    insert_subprocess_mock(
+    insert_subprocess_check_output_mock(
         (
             'attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar',
             '--verbose', '--stats',
@@ -53,6 +88,39 @@ def test_create_archive_with_verbose_should_call_attic_with_verbose_parameters()
         repository='repo',
     )
 
+def test_create_archive_with_missing_repository_should_raise():
+    insert_subprocess_check_output_mock(
+        ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar'),
+        error_output='Error: Repository repo does not exist',
+    )
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    with assert_raises(RuntimeError):
+        module.create_archive(
+            excludes_filename='excludes',
+            verbose=False,
+            source_directories='foo bar',
+            repository='repo',
+        )
+
+
+def test_create_archive_with_other_error_should_raise():
+    subprocess = insert_subprocess_check_output_mock(
+        ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar'),
+        error_output='Something went wrong',
+    )
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    with assert_raises(subprocess.CalledProcessError):
+        module.create_archive(
+            excludes_filename='excludes',
+            verbose=False,
+            source_directories='foo bar',
+            repository='repo',
+        )
+
 
 BASE_PRUNE_FLAGS = (
     ('--keep-daily', '1'),
@@ -80,7 +148,7 @@ def test_prune_archives_should_call_attic_with_parameters():
     flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS,
     )
-    insert_subprocess_mock(
+    insert_subprocess_check_call_mock(
         (
             'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly',
             '3',
@@ -99,7 +167,7 @@ def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters()
     flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return(
         BASE_PRUNE_FLAGS,
     )
-    insert_subprocess_mock(
+    insert_subprocess_check_call_mock(
         (
             'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly',
             '3', '--verbose',
@@ -111,3 +179,34 @@ def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters()
         verbose=True,
         retention_config=retention_config,
     )
+
+
+def test_check_archives_should_call_attic_with_parameters():
+    stdout = flexmock()
+    insert_subprocess_check_call_mock(
+        ('attic', 'check', 'repo'),
+        stdout=stdout,
+    )
+    insert_platform_mock()
+    insert_datetime_mock()
+    flexmock(module).open = lambda filename, mode: stdout
+    flexmock(module).os = flexmock().should_receive('devnull').mock
+
+    module.check_archives(
+        verbose=False,
+        repository='repo',
+    )
+
+
+def test_check_archives_with_verbose_should_call_attic_with_verbose_parameters():
+    insert_subprocess_check_call_mock(
+        ('attic', 'check', 'repo', '--verbose'),
+        stdout=None,
+    )
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.check_archives(
+        verbose=True,
+        repository='repo',
+    )

setup.cfg

@@ -0,0 +1,2 @@
+[nosetests]
+detailed-errors=1

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='atticmatic',
-    version='0.0.2',
+    version='0.0.4',
     description='A wrapper script for Attic backup software that creates and prunes backups',
     author='Dan Helfman',
     author_email='witten@torsion.org',

test_requirements.txt

@@ -0,0 +1,2 @@
+flexmock==0.9.7
+nose==1.3.4

tox.ini

@@ -0,0 +1,8 @@
+[tox]
+envlist=py27,py34
+skipsdist=True
+
+[testenv]
+usedevelop=True
+deps=-rtest_requirements.txt
+commands = nosetests