Compare commits

...
7 Commits
14 changed files with 381 additions and 91 deletions
+3
View File
@@ -6,3 +6,6 @@ b31d51b633701554e84f996cc0c73bad2990780b 0.0.5
aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5
aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5
569aef47a9b25c55b13753f94706f5d330219995 0.0.5
569aef47a9b25c55b13753f94706f5d330219995 0.0.5
a03495a8e8b471da63b5e2ae79d3ff9065839c2a 0.0.5
7ea93ca83f426ec0a608a68580c72c0775b81f86 0.0.6
+9
View File
@@ -1,3 +1,12 @@
0.0.7
* Flag for multiple levels of verbosity: some, and lots.
* Improved mocking of Python builtins in unit tests.
0.0.6
* New configuration section for customizing which Attic consistency checks run, if any.
0.0.5
* Fixed regression with --verbose output being buffered. This means dropping the helpful error
+10 -2
View File
@@ -26,6 +26,10 @@ Here's an example config file:
keep_weekly: 4
keep_monthly: 6
[consistency]
# Consistency checks to run, or "disabled" to prevent checks.
checks: repository archives
Additionally, exclude patterns can be specified in a separate excludes config
file, one pattern per line.
@@ -74,9 +78,13 @@ and check backups for consistency problems due to things like file damage.
By default, the backup will proceed silently except in the case of errors. But
if you'd like to to get additional information about the progress of the
backup as it proceeds, use the verbose option instead:
backup as it proceeds, use the verbosity option:
atticmattic --verbose
atticmattic --verbosity 1
Or, for even more progress spew:
atticmattic --verbosity 2
If you'd like to see the available command-line arguments, view the help:
+82 -13
View File
@@ -3,13 +3,19 @@ import os
import platform
import subprocess
from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
def create_archive(excludes_filename, verbose, source_directories, repository):
def create_archive(excludes_filename, verbosity, source_directories, repository):
'''
Given an excludes filename, a vebosity flag, a space-separated list of source directories, and
a local or remote repository path, create an attic archive.
'''
sources = tuple(source_directories.split(' '))
verbosity_flags = {
VERBOSITY_SOME: ('--stats',),
VERBOSITY_LOTS: ('--verbose', '--stats'),
}.get(verbosity, ())
command = (
'attic', 'create',
@@ -19,14 +25,12 @@ def create_archive(excludes_filename, verbose, source_directories, repository):
hostname=platform.node(),
timestamp=datetime.now().isoformat(),
),
) + sources + (
('--verbose', '--stats') if verbose else ()
)
) + sources + verbosity_flags
subprocess.check_call(command)
def make_prune_flags(retention_config):
def _make_prune_flags(retention_config):
'''
Given a retention config dict mapping from option name to value, tranform it into an iterable of
command-line name-value flag pairs.
@@ -48,34 +52,99 @@ def make_prune_flags(retention_config):
)
def prune_archives(verbose, repository, retention_config):
def prune_archives(verbosity, repository, retention_config):
'''
Given a verbosity flag, a local or remote repository path, and a retention config dict, prune
attic archives according the the retention policy specified in that configuration.
'''
verbosity_flags = {
VERBOSITY_SOME: ('--stats',),
VERBOSITY_LOTS: ('--verbose', '--stats'),
}.get(verbosity, ())
command = (
'attic', 'prune',
repository,
) + tuple(
element
for pair in make_prune_flags(retention_config)
for pair in _make_prune_flags(retention_config)
for element in pair
) + (('--verbose',) if verbose else ())
) + verbosity_flags
subprocess.check_call(command)
def check_archives(verbose, repository):
DEFAULT_CHECKS = ('repository', 'archives')
def _parse_checks(consistency_config):
'''
Given a verbosity flag and a local or remote repository path, check the contained attic archives
for consistency.
Given a consistency config with a space-separated "checks" option, transform it to a tuple of
named checks to run.
For example, given a retention config of:
{'checks': 'repository archives'}
This will be returned as:
('repository', 'archives')
If no "checks" option is present, return the DEFAULT_CHECKS. If the checks value is the string
"disabled", return an empty tuple, meaning that no checks should be run.
'''
checks = consistency_config.get('checks', '').strip()
if not checks:
return DEFAULT_CHECKS
return tuple(
check for check in consistency_config['checks'].split(' ')
if check.lower() not in ('disabled', '')
)
def _make_check_flags(checks):
'''
Given a parsed sequence of checks, transform it into tuple of command-line flags.
For example, given parsed checks of:
('repository',)
This will be returned as:
('--repository-only',)
'''
if checks == DEFAULT_CHECKS:
return ()
return tuple(
'--{}-only'.format(check) for check in checks
)
def check_archives(verbosity, repository, consistency_config):
'''
Given a verbosity flag, a local or remote repository path, and a consistency config dict, check
the contained attic archives for consistency.
If there are no consistency checks to run, skip running them.
'''
checks = _parse_checks(consistency_config)
if not checks:
return
verbosity_flags = {
VERBOSITY_SOME: ('--verbose',),
VERBOSITY_LOTS: ('--verbose',),
}.get(verbosity, ())
command = (
'attic', 'check',
repository,
) + (('--verbose',) if verbose else ())
) + _make_check_flags(checks) + verbosity_flags
# Attic's check command spews to stdout even without the verbose flag. Suppress it.
stdout = None if verbose else open(os.devnull, 'w')
stdout = None if verbosity_flags else open(os.devnull, 'w')
subprocess.check_call(command, stdout=stdout)
+8 -8
View File
@@ -29,9 +29,9 @@ def parse_arguments(*arguments):
help='Excludes filename',
)
parser.add_argument(
'-v', '--verbose',
action='store_true',
help='Display verbose progress information',
'-v', '--verbosity',
type=int,
help='Display verbose progress (1 for some, 2 for lots)',
)
return parser.parse_args(arguments)
@@ -40,12 +40,12 @@ def parse_arguments(*arguments):
def main():
try:
args = parse_arguments(*sys.argv[1:])
location_config, retention_config = parse_configuration(args.config_filename)
repository = location_config['repository']
config = parse_configuration(args.config_filename)
repository = config.location['repository']
create_archive(args.excludes_filename, args.verbose, **location_config)
prune_archives(args.verbose, repository, retention_config)
check_archives(args.verbose, repository)
create_archive(args.excludes_filename, args.verbosity, **config.location)
prune_archives(args.verbosity, repository, config.retention)
check_archives(args.verbosity, repository, config.consistency)
except (ValueError, IOError, CalledProcessError) as error:
print(error, file=sys.stderr)
sys.exit(1)
+40 -13
View File
@@ -39,6 +39,12 @@ CONFIG_FORMAT = (
option('keep_yearly', int, required=False),
option('prefix', required=False),
),
),
Section_format(
'consistency',
(
option('checks', required=False),
),
)
)
@@ -49,20 +55,34 @@ def validate_configuration_format(parser, config_format):
configuration file has the expected sections, that any required options are present in those
sections, and that there aren't any unexpected options.
A section is required if any of its contained options are required.
Raise ValueError if anything is awry.
'''
section_names = parser.sections()
required_section_names = tuple(section.name for section in config_format)
section_names = set(parser.sections())
required_section_names = tuple(
section.name for section in config_format
if any(option.required for option in section.options)
)
if set(section_names) != set(required_section_names):
unknown_section_names = section_names - set(
section_format.name for section_format in config_format
)
if unknown_section_names:
raise ValueError(
'Expected config sections {} but found sections: {}'.format(
', '.join(required_section_names),
', '.join(section_names)
)
'Unknown config sections found: {}'.format(', '.join(unknown_section_names))
)
missing_section_names = set(required_section_names) - section_names
if missing_section_names:
raise ValueError(
'Missing config sections: {}'.format(', '.join(missing_section_names))
)
for section_format in config_format:
if section_format.name not in section_names:
continue
option_names = parser.options(section_format.name)
expected_options = section_format.options
@@ -90,6 +110,11 @@ def validate_configuration_format(parser, config_format):
)
# Describes a parsed configuration, where each attribute is the name of a configuration file section
# and each value is a dict of that section's parsed options.
Parsed_config = namedtuple('Config', (section_format.name for section_format in CONFIG_FORMAT))
def parse_section_options(parser, section_format):
'''
Given an open ConfigParser and an expected section format, return the option values from that
@@ -112,17 +137,19 @@ def parse_section_options(parser, section_format):
def parse_configuration(config_filename):
'''
Given a config filename of the expected format, return the parsed configuration as a tuple of
(location config, retention config) where each config is a dict of that section's options.
Given a config filename of the expected format, return the parsed configuration as Parsed_config
data structure.
Raise IOError if the file cannot be read, or ValueError if the format is not as expected.
'''
parser = ConfigParser()
parser.readfp(open(config_filename))
parser.read(config_filename)
validate_configuration_format(parser, CONFIG_FORMAT)
return tuple(
parse_section_options(parser, section_format)
for section_format in CONFIG_FORMAT
return Parsed_config(
*(
parse_section_options(parser, section_format)
for section_format in CONFIG_FORMAT
)
)
+11
View File
@@ -0,0 +1,11 @@
from flexmock import flexmock
import sys
def builtins_mock():
try:
# Python 2
return flexmock(sys.modules['__builtin__'])
except KeyError:
# Python 3
return flexmock(sys.modules['builtins'])
+5 -5
View File
@@ -10,7 +10,7 @@ def test_parse_arguments_with_no_arguments_uses_defaults():
assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME
assert parser.verbose == False
assert parser.verbosity == None
def test_parse_arguments_with_filename_arguments_overrides_defaults():
@@ -18,15 +18,15 @@ def test_parse_arguments_with_filename_arguments_overrides_defaults():
assert parser.config_filename == 'myconfig'
assert parser.excludes_filename == 'myexcludes'
assert parser.verbose == False
assert parser.verbosity == None
def test_parse_arguments_with_verbose_flag_overrides_default():
parser = module.parse_arguments('--verbose')
def test_parse_arguments_with_verbosity_flag_overrides_default():
parser = module.parse_arguments('--verbosity', '1')
assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME
assert parser.verbose == True
assert parser.verbosity == 1
def test_parse_arguments_with_invalid_arguments_exits():
+136 -36
View File
@@ -3,6 +3,8 @@ from collections import OrderedDict
from flexmock import flexmock
from atticmatic import attic as module
from atticmatic.tests.builtins import builtins_mock
from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
def insert_subprocess_mock(check_call_command, **kwargs):
@@ -11,8 +13,14 @@ def insert_subprocess_mock(check_call_command, **kwargs):
flexmock(module).subprocess = subprocess
def insert_subprocess_never():
subprocess = flexmock()
subprocess.should_receive('check_call').never()
flexmock(module).subprocess = subprocess
def insert_platform_mock():
flexmock(module).platform = flexmock().should_receive('node').and_return('host').mock
flexmock(module.platform).should_receive('node').and_return('host')
def insert_datetime_mock():
@@ -21,34 +29,43 @@ def insert_datetime_mock():
).mock
CREATE_COMMAND = ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar')
def test_create_archive_should_call_attic_with_parameters():
insert_subprocess_mock(
('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar'),
)
insert_subprocess_mock(CREATE_COMMAND)
insert_platform_mock()
insert_datetime_mock()
module.create_archive(
excludes_filename='excludes',
verbose=False,
verbosity=None,
source_directories='foo bar',
repository='repo',
)
def test_create_archive_with_verbose_should_call_attic_with_verbose_parameters():
insert_subprocess_mock(
(
'attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar',
'--verbose', '--stats',
),
)
def test_create_archive_with_verbosity_some_should_call_attic_with_stats_parameter():
insert_subprocess_mock(CREATE_COMMAND + ('--stats',))
insert_platform_mock()
insert_datetime_mock()
module.create_archive(
excludes_filename='excludes',
verbose=True,
verbosity=VERBOSITY_SOME,
source_directories='foo bar',
repository='repo',
)
def test_create_archive_with_verbosity_lots_should_call_attic_with_verbose_parameter():
insert_subprocess_mock(CREATE_COMMAND + ('--verbose', '--stats'))
insert_platform_mock()
insert_datetime_mock()
module.create_archive(
excludes_filename='excludes',
verbosity=VERBOSITY_LOTS,
source_directories='foo bar',
repository='repo',
)
@@ -70,50 +87,98 @@ def test_make_prune_flags_should_return_flags_from_config():
)
)
result = module.make_prune_flags(retention_config)
result = module._make_prune_flags(retention_config)
assert tuple(result) == BASE_PRUNE_FLAGS
PRUNE_COMMAND = (
'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3',
)
def test_prune_archives_should_call_attic_with_parameters():
retention_config = flexmock()
flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return(
flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
BASE_PRUNE_FLAGS,
)
insert_subprocess_mock(
(
'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly',
'3',
),
)
insert_subprocess_mock(PRUNE_COMMAND)
module.prune_archives(
verbose=False,
verbosity=None,
repository='repo',
retention_config=retention_config,
)
def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters():
def test_prune_archives_with_verbosity_some_should_call_attic_with_stats_parameter():
retention_config = flexmock()
flexmock(module).should_receive('make_prune_flags').with_args(retention_config).and_return(
flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
BASE_PRUNE_FLAGS,
)
insert_subprocess_mock(
(
'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly',
'3', '--verbose',
),
)
insert_subprocess_mock(PRUNE_COMMAND + ('--stats',))
module.prune_archives(
repository='repo',
verbose=True,
verbosity=VERBOSITY_SOME,
retention_config=retention_config,
)
def test_prune_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter():
retention_config = flexmock()
flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
BASE_PRUNE_FLAGS,
)
insert_subprocess_mock(PRUNE_COMMAND + ('--verbose', '--stats',))
module.prune_archives(
repository='repo',
verbosity=VERBOSITY_LOTS,
retention_config=retention_config,
)
def test_parse_checks_returns_them_as_tuple():
checks = module._parse_checks({'checks': 'foo disabled bar'})
assert checks == ('foo', 'bar')
def test_parse_checks_with_missing_value_returns_defaults():
checks = module._parse_checks({})
assert checks == module.DEFAULT_CHECKS
def test_parse_checks_with_blank_value_returns_defaults():
checks = module._parse_checks({'checks': ''})
assert checks == module.DEFAULT_CHECKS
def test_parse_checks_with_disabled_returns_no_checks():
checks = module._parse_checks({'checks': 'disabled'})
assert checks == ()
def test_make_check_flags_with_checks_returns_flags():
flags = module._make_check_flags(('foo', 'bar'))
assert flags == ('--foo-only', '--bar-only')
def test_make_check_flags_with_default_checks_returns_no_flags():
flags = module._make_check_flags(module.DEFAULT_CHECKS)
assert flags == ()
def test_check_archives_should_call_attic_with_parameters():
consistency_config = flexmock()
flexmock(module).should_receive('_parse_checks').and_return(flexmock())
flexmock(module).should_receive('_make_check_flags').and_return(())
stdout = flexmock()
insert_subprocess_mock(
('attic', 'check', 'repo'),
@@ -121,16 +186,20 @@ def test_check_archives_should_call_attic_with_parameters():
)
insert_platform_mock()
insert_datetime_mock()
flexmock(module).open = lambda filename, mode: stdout
flexmock(module).os = flexmock().should_receive('devnull').mock
builtins_mock().should_receive('open').and_return(stdout)
flexmock(module.os).should_receive('devnull')
module.check_archives(
verbose=False,
verbosity=None,
repository='repo',
consistency_config=consistency_config,
)
def test_check_archives_with_verbose_should_call_attic_with_verbose_parameters():
def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_parameter():
consistency_config = flexmock()
flexmock(module).should_receive('_parse_checks').and_return(flexmock())
flexmock(module).should_receive('_make_check_flags').and_return(())
insert_subprocess_mock(
('attic', 'check', 'repo', '--verbose'),
stdout=None,
@@ -139,6 +208,37 @@ def test_check_archives_with_verbose_should_call_attic_with_verbose_parameters()
insert_datetime_mock()
module.check_archives(
verbose=True,
verbosity=VERBOSITY_SOME,
repository='repo',
consistency_config=consistency_config,
)
def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter():
consistency_config = flexmock()
flexmock(module).should_receive('_parse_checks').and_return(flexmock())
flexmock(module).should_receive('_make_check_flags').and_return(())
insert_subprocess_mock(
('attic', 'check', 'repo', '--verbose'),
stdout=None,
)
insert_platform_mock()
insert_datetime_mock()
module.check_archives(
verbosity=VERBOSITY_LOTS,
repository='repo',
consistency_config=consistency_config,
)
def test_check_archives_without_any_checks_should_bail():
consistency_config = flexmock()
flexmock(module).should_receive('_parse_checks').and_return(())
insert_subprocess_never()
module.check_archives(
verbosity=None,
repository='repo',
consistency_config=consistency_config,
)
+65 -10
View File
@@ -41,19 +41,55 @@ def test_validate_configuration_format_with_valid_config_should_not_raise():
module.validate_configuration_format(parser, config_format)
def test_validate_configuration_format_with_missing_section_should_raise():
def test_validate_configuration_format_with_missing_required_section_should_raise():
parser = flexmock()
parser.should_receive('sections').and_return(('section',))
config_format = (
module.Section_format('section', options=()),
module.Section_format('missing', options=()),
module.Section_format(
'section',
options=(
module.Config_option('stuff', str, required=True),
),
),
# At least one option in this section is required, so the section is required.
module.Section_format(
'missing',
options=(
module.Config_option('such', str, required=False),
module.Config_option('things', str, required=True),
),
),
)
with assert_raises(ValueError):
module.validate_configuration_format(parser, config_format)
def test_validate_configuration_format_with_extra_section_should_raise():
def test_validate_configuration_format_with_missing_optional_section_should_not_raise():
parser = flexmock()
parser.should_receive('sections').and_return(('section',))
parser.should_receive('options').with_args('section').and_return(('stuff',))
config_format = (
module.Section_format(
'section',
options=(
module.Config_option('stuff', str, required=True),
),
),
# No options in the section are required, so the section is optional.
module.Section_format(
'missing',
options=(
module.Config_option('such', str, required=False),
module.Config_option('things', str, required=False),
),
),
)
module.validate_configuration_format(parser, config_format)
def test_validate_configuration_format_with_unknown_section_should_raise():
parser = flexmock()
parser.should_receive('sections').and_return(('section', 'extra'))
config_format = (
@@ -139,11 +175,30 @@ def test_parse_section_options_should_return_section_options():
)
def test_parse_section_options_for_missing_section_should_return_empty_dict():
parser = flexmock()
parser.should_receive('get').never()
parser.should_receive('getint').never()
parser.should_receive('has_option').with_args('section', 'foo').and_return(False)
parser.should_receive('has_option').with_args('section', 'bar').and_return(False)
section_format = module.Section_format(
'section',
(
module.Config_option('foo', str, required=False),
module.Config_option('bar', int, required=False),
),
)
config = module.parse_section_options(parser, section_format)
assert config == OrderedDict()
def insert_mock_parser():
parser = flexmock()
parser.should_receive('readfp')
flexmock(module).open = lambda filename: None
flexmock(module).ConfigParser = parser
parser.should_receive('read')
module.ConfigParser = lambda: parser
return parser
@@ -154,13 +209,13 @@ def test_parse_configuration_should_return_section_configs():
mock_module.should_receive('validate_configuration_format').with_args(
parser, module.CONFIG_FORMAT,
).once()
mock_section_configs = (flexmock(), flexmock())
mock_section_configs = (flexmock(),) * len(module.CONFIG_FORMAT)
for section_format, section_config in zip(module.CONFIG_FORMAT, mock_section_configs):
mock_module.should_receive('parse_section_options').with_args(
parser, section_format,
).and_return(section_config).once()
section_configs = module.parse_configuration('filename')
parsed_config = module.parse_configuration('filename')
assert section_configs == mock_section_configs
assert parsed_config == module.Parsed_config(*mock_section_configs)
+2
View File
@@ -0,0 +1,2 @@
VERBOSITY_SOME = 1
VERBOSITY_LOTS = 2
+8 -2
View File
@@ -6,8 +6,8 @@ source_directories: /home /etc
repository: user@backupserver:sourcehostname.attic
[retention]
# Retention policy for how many backups to keep in each category.
# See https://attic-backup.org/usage.html#attic-prune for details.
# Retention policy for how many backups to keep in each category. See
# https://attic-backup.org/usage.html#attic-prune for details.
#keep_within: 3h
#keep_hourly: 24
keep_daily: 7
@@ -15,3 +15,9 @@ keep_weekly: 4
keep_monthly: 6
keep_yearly: 1
#prefix: sourcehostname
[consistency]
# Space-separated list of consistency checks to run: "repository", "archives",
# or both. Defaults to both. Set to "disabled" to disable all consistency
# checks. See https://attic-backup.org/usage.html#attic-check for details.
checks: repository archives
+1 -1
View File
@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
setup(
name='atticmatic',
version='0.0.5',
version='0.0.7',
description='A wrapper script for Attic backup software that creates and prunes backups',
author='Dan Helfman',
author_email='witten@torsion.org',
+1 -1
View File
@@ -5,4 +5,4 @@ skipsdist=True
[testenv]
usedevelop=True
deps=-rtest_requirements.txt
commands = nosetests
commands = nosetests []