Compare commits

...
6 Commits
18 changed files with 304 additions and 113 deletions
+2
View File
@@ -8,3 +8,5 @@ aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5
569aef47a9b25c55b13753f94706f5d330219995 0.0.5
569aef47a9b25c55b13753f94706f5d330219995 0.0.5
a03495a8e8b471da63b5e2ae79d3ff9065839c2a 0.0.5
7ea93ca83f426ec0a608a68580c72c0775b81f86 0.0.6
cf4c7065f0711deda1cba878398bc05390e2c3f9 0.0.7
+9
View File
@@ -1,3 +1,12 @@
0.1.0
* New "borgmatic" command to support Borg backup software, a fork of Attic.
0.0.7
* Flag for multiple levels of verbosity: some, and lots.
* Improved mocking of Python builtins in unit tests.
0.0.6
* New configuration section for customizing which Attic consistency checks run, if any.
+33 -16
View File
@@ -4,12 +4,13 @@ save_as: atticmatic/index.html
## Overview
atticmatic is a simple Python wrapper script for the [Attic backup
software](https://attic-backup.org/) that initiates a backup, prunes any old
backups according to a retention policy, and validates backups for
consistency. The script supports specifying your settings in a declarative
configuration file rather than having to put them all on the command-line, and
handles common errors.
atticmatic is a simple Python wrapper script for the
[Attic](https://attic-backup.org/) and
[Borg](https://borgbackup.github.io/borgbackup/) backup software that
initiates a backup, prunes any old backups according to a retention policy,
and validates backups for consistency. The script supports specifying your
settings in a declarative configuration file rather than having to put them
all on the command-line, and handles common errors.
Here's an example config file:
@@ -17,7 +18,7 @@ Here's an example config file:
# Space-separated list of source directories to backup.
source_directories: /home /etc
# Path to local or remote Attic repository.
# Path to local or remote backup repository.
repository: user@backupserver:sourcehostname.attic
[retention]
@@ -27,6 +28,7 @@ Here's an example config file:
keep_monthly: 6
[consistency]
# Consistency checks to run, or "disabled" to prevent checks.
checks: repository archives
Additionally, exclude patterns can be specified in a separate excludes config
@@ -40,14 +42,14 @@ available](https://torsion.org/hg/atticmatic). It's also mirrored on
## Setup
To get up and running with Attic, follow the [Attic Quick
Start](https://attic-backup.org/quickstart.html) guide to create an Attic
To get up and running, follow the [Attic Quick
Start](https://attic-backup.org/quickstart.html) or the [Borg Quick
Start](https://borgbackup.github.io/borgbackup/quickstart.html) to create a
repository on a local or remote host. Note that if you plan to run atticmatic
on a schedule with cron, and you encrypt your attic repository with a
passphrase instead of a key file, you'll need to set the `ATTIC_PASSPHRASE`
environment variable. See [attic's repository encryption
documentation](https://attic-backup.org/quickstart.html#encrypted-repos) for
more info.
environment variable. See the repository encryption section of the Quick Start
for more info.
If the repository is on a remote host, make sure that your local root user has
key-based ssh access to the desired user account on the remote host.
@@ -56,13 +58,19 @@ To install atticmatic, run the following command to download and install it:
sudo pip install --upgrade hg+https://torsion.org/hg/atticmatic
Then copy the following configuration files:
If you are using Attic, copy the following configuration files:
sudo cp sample/atticmatic.cron /etc/cron.d/atticmatic
sudo mkdir /etc/atticmatic/
sudo cp sample/config sample/excludes /etc/atticmatic/
Lastly, modify those files with your desired configuration.
If you are using Borg, copy the files like this instead:
sudo cp sample/atticmatic.cron /etc/cron.d/borgmatic
sudo mkdir /etc/borgmatic/
sudo cp sample/config sample/excludes /etc/borgmatic/
Lastly, modify the /etc files with your desired configuration.
## Usage
@@ -72,14 +80,23 @@ arguments:
atticmatic
Or, if you're using Borg, use this command instead to make use of the Borg
backend:
borgmatic
This will also prune any old backups as per the configured retention policy,
and check backups for consistency problems due to things like file damage.
By default, the backup will proceed silently except in the case of errors. But
if you'd like to to get additional information about the progress of the
backup as it proceeds, use the verbose option instead:
backup as it proceeds, use the verbosity option:
atticmattic --verbose
atticmattic --verbosity 1
Or, for even more progress spew:
atticmattic --verbosity 2
If you'd like to see the available command-line arguments, view the help:
View File
+12
View File
@@ -0,0 +1,12 @@
from functools import partial
from atticmatic.backends import shared
# An atticmatic backend that supports Attic for actually handling backups.
COMMAND = 'attic'
create_archive = partial(shared.create_archive, command=COMMAND)
prune_archives = partial(shared.prune_archives, command=COMMAND)
check_archives = partial(shared.check_archives, command=COMMAND)
+13
View File
@@ -0,0 +1,13 @@
from functools import partial
from atticmatic.backends import shared
# An atticmatic backend that supports Borg for actually handling backups.
COMMAND = 'borg'
create_archive = partial(shared.create_archive, command=COMMAND)
prune_archives = partial(shared.prune_archives, command=COMMAND)
check_archives = partial(shared.check_archives, command=COMMAND)
@@ -3,27 +3,37 @@ import os
import platform
import subprocess
from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
def create_archive(excludes_filename, verbose, source_directories, repository):
# Common backend functionality shared by Attic and Borg. As the two backup
# commands diverge, these shared functions will likely need to be replaced
# with non-shared functions within atticmatic.backends.attic and
# atticmatic.backends.borg.
def create_archive(excludes_filename, verbosity, source_directories, repository, command):
'''
Given an excludes filename, a vebosity flag, a space-separated list of source directories, and
a local or remote repository path, create an attic archive.
Given an excludes filename, a vebosity flag, a space-separated list of source directories, a
local or remote repository path, and a command to run, create an attic archive.
'''
sources = tuple(source_directories.split(' '))
verbosity_flags = {
VERBOSITY_SOME: ('--stats',),
VERBOSITY_LOTS: ('--verbose', '--stats'),
}.get(verbosity, ())
command = (
'attic', 'create',
full_command = (
command, 'create',
'--exclude-from', excludes_filename,
'{repo}::{hostname}-{timestamp}'.format(
repo=repository,
hostname=platform.node(),
timestamp=datetime.now().isoformat(),
),
) + sources + (
('--verbose', '--stats') if verbose else ()
)
) + sources + verbosity_flags
subprocess.check_call(command)
subprocess.check_call(full_command)
def _make_prune_flags(retention_config):
@@ -48,21 +58,27 @@ def _make_prune_flags(retention_config):
)
def prune_archives(verbose, repository, retention_config):
def prune_archives(verbosity, repository, retention_config, command):
'''
Given a verbosity flag, a local or remote repository path, and a retention config dict, prune
attic archives according the the retention policy specified in that configuration.
Given a verbosity flag, a local or remote repository path, a retention config dict, and a
command to run, prune attic archives according the the retention policy specified in that
configuration.
'''
command = (
'attic', 'prune',
verbosity_flags = {
VERBOSITY_SOME: ('--stats',),
VERBOSITY_LOTS: ('--verbose', '--stats'),
}.get(verbosity, ())
full_command = (
command, 'prune',
repository,
) + tuple(
element
for pair in _make_prune_flags(retention_config)
for element in pair
) + (('--verbose',) if verbose else ())
) + verbosity_flags
subprocess.check_call(command)
subprocess.check_call(full_command)
DEFAULT_CHECKS = ('repository', 'archives')
@@ -114,10 +130,10 @@ def _make_check_flags(checks):
)
def check_archives(verbose, repository, consistency_config):
def check_archives(verbosity, repository, consistency_config, command):
'''
Given a verbosity flag, a local or remote repository path, and a consistency config dict, check
the contained attic archives for consistency.
Given a verbosity flag, a local or remote repository path, a consistency config dict, and a
command to run, check the contained attic archives for consistency.
If there are no consistency checks to run, skip running them.
'''
@@ -125,12 +141,17 @@ def check_archives(verbose, repository, consistency_config):
if not checks:
return
command = (
'attic', 'check',
verbosity_flags = {
VERBOSITY_SOME: ('--verbose',),
VERBOSITY_LOTS: ('--verbose',),
}.get(verbosity, ())
full_command = (
command, 'check',
repository,
) + _make_check_flags(checks) + (('--verbose',) if verbose else ())
) + _make_check_flags(checks) + verbosity_flags
# Attic's check command spews to stdout even without the verbose flag. Suppress it.
stdout = None if verbose else open(os.devnull, 'w')
# The check command spews to stdout even without the verbose flag. Suppress it.
stdout = None if verbosity_flags else open(os.devnull, 'w')
subprocess.check_call(command, stdout=stdout)
subprocess.check_call(full_command, stdout=stdout)
+32 -14
View File
@@ -1,51 +1,69 @@
from __future__ import print_function
from argparse import ArgumentParser
from importlib import import_module
import os
from subprocess import CalledProcessError
import sys
from atticmatic.attic import check_archives, create_archive, prune_archives
from atticmatic.config import parse_configuration
DEFAULT_CONFIG_FILENAME = '/etc/atticmatic/config'
DEFAULT_EXCLUDES_FILENAME = '/etc/atticmatic/excludes'
DEFAULT_CONFIG_FILENAME_PATTERN = '/etc/{}/config'
DEFAULT_EXCLUDES_FILENAME_PATTERN = '/etc/{}/excludes'
def parse_arguments(*arguments):
def parse_arguments(command_name, *arguments):
'''
Parse the given command-line arguments and return them as an ArgumentParser instance.
Given the name of the command with which this script was invoked and command-line arguments,
parse the arguments and return them as an ArgumentParser instance. Use the command name to
determine the default configuration and excludes paths.
'''
parser = ArgumentParser()
parser.add_argument(
'-c', '--config',
dest='config_filename',
default=DEFAULT_CONFIG_FILENAME,
default=DEFAULT_CONFIG_FILENAME_PATTERN.format(command_name),
help='Configuration filename',
)
parser.add_argument(
'--excludes',
dest='excludes_filename',
default=DEFAULT_EXCLUDES_FILENAME,
default=DEFAULT_EXCLUDES_FILENAME_PATTERN.format(command_name),
help='Excludes filename',
)
parser.add_argument(
'-v', '--verbose',
action='store_true',
help='Display verbose progress information',
'-v', '--verbosity',
type=int,
help='Display verbose progress (1 for some, 2 for lots)',
)
return parser.parse_args(arguments)
def load_backend(command_name):
'''
Given the name of the command with which this script was invoked, return the corresponding
backend module responsible for actually dealing with backups.
'''
backend_name = {
'atticmatic': 'attic',
'borgmatic': 'borg',
}.get(command_name, 'attic')
return import_module('atticmatic.backends.{}'.format(backend_name))
def main():
try:
args = parse_arguments(*sys.argv[1:])
command_name = os.path.basename(sys.argv[0])
args = parse_arguments(command_name, *sys.argv[1:])
config = parse_configuration(args.config_filename)
repository = config.location['repository']
backend = load_backend(command_name)
create_archive(args.excludes_filename, args.verbose, **config.location)
prune_archives(args.verbose, repository, config.retention)
check_archives(args.verbose, repository, config.consistency)
backend.create_archive(args.excludes_filename, args.verbosity, **config.location)
backend.prune_archives(args.verbosity, repository, config.retention)
backend.check_archives(args.verbosity, repository, config.consistency)
except (ValueError, IOError, CalledProcessError) as error:
print(error, file=sys.stderr)
sys.exit(1)
+1 -1
View File
@@ -143,7 +143,7 @@ def parse_configuration(config_filename):
Raise IOError if the file cannot be read, or ValueError if the format is not as expected.
'''
parser = ConfigParser()
parser.readfp(open(config_filename))
parser.read(config_filename)
validate_configuration_format(parser, CONFIG_FORMAT)
+11
View File
@@ -0,0 +1,11 @@
from flexmock import flexmock
import sys
def builtins_mock():
try:
# Python 2
return flexmock(sys.modules['__builtin__'])
except KeyError:
# Python 3
return flexmock(sys.modules['builtins'])
+16 -13
View File
@@ -5,28 +5,31 @@ from nose.tools import assert_raises
from atticmatic import command as module
def test_parse_arguments_with_no_arguments_uses_defaults():
parser = module.parse_arguments()
COMMAND_NAME = 'foomatic'
assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME
assert parser.verbose == False
def test_parse_arguments_with_no_arguments_uses_defaults():
parser = module.parse_arguments(COMMAND_NAME)
assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME)
assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME_PATTERN.format(COMMAND_NAME)
assert parser.verbosity == None
def test_parse_arguments_with_filename_arguments_overrides_defaults():
parser = module.parse_arguments('--config', 'myconfig', '--excludes', 'myexcludes')
parser = module.parse_arguments(COMMAND_NAME, '--config', 'myconfig', '--excludes', 'myexcludes')
assert parser.config_filename == 'myconfig'
assert parser.excludes_filename == 'myexcludes'
assert parser.verbose == False
assert parser.verbosity == None
def test_parse_arguments_with_verbose_flag_overrides_default():
parser = module.parse_arguments('--verbose')
def test_parse_arguments_with_verbosity_flag_overrides_default():
parser = module.parse_arguments(COMMAND_NAME, '--verbosity', '1')
assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME
assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME
assert parser.verbose == True
assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME)
assert parser.excludes_filename == module.DEFAULT_EXCLUDES_FILENAME_PATTERN.format(COMMAND_NAME)
assert parser.verbosity == 1
def test_parse_arguments_with_invalid_arguments_exits():
@@ -35,6 +38,6 @@ def test_parse_arguments_with_invalid_arguments_exits():
try:
with assert_raises(SystemExit):
module.parse_arguments('--posix-me-harder')
module.parse_arguments(COMMAND_NAME, '--posix-me-harder')
finally:
sys.stderr = original_stderr
@@ -2,7 +2,9 @@ from collections import OrderedDict
from flexmock import flexmock
from atticmatic import attic as module
from atticmatic.backends import shared as module
from atticmatic.tests.builtins import builtins_mock
from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
def insert_subprocess_mock(check_call_command, **kwargs):
@@ -18,7 +20,7 @@ def insert_subprocess_never():
def insert_platform_mock():
flexmock(module).platform = flexmock().should_receive('node').and_return('host').mock
flexmock(module.platform).should_receive('node').and_return('host')
def insert_datetime_mock():
@@ -27,36 +29,48 @@ def insert_datetime_mock():
).mock
CREATE_COMMAND = ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar')
def test_create_archive_should_call_attic_with_parameters():
insert_subprocess_mock(
('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar'),
)
insert_subprocess_mock(CREATE_COMMAND)
insert_platform_mock()
insert_datetime_mock()
module.create_archive(
excludes_filename='excludes',
verbose=False,
verbosity=None,
source_directories='foo bar',
repository='repo',
command='attic',
)
def test_create_archive_with_verbose_should_call_attic_with_verbose_parameters():
insert_subprocess_mock(
(
'attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar',
'--verbose', '--stats',
),
)
def test_create_archive_with_verbosity_some_should_call_attic_with_stats_parameter():
insert_subprocess_mock(CREATE_COMMAND + ('--stats',))
insert_platform_mock()
insert_datetime_mock()
module.create_archive(
excludes_filename='excludes',
verbose=True,
verbosity=VERBOSITY_SOME,
source_directories='foo bar',
repository='repo',
command='attic',
)
def test_create_archive_with_verbosity_lots_should_call_attic_with_verbose_parameter():
insert_subprocess_mock(CREATE_COMMAND + ('--verbose', '--stats'))
insert_platform_mock()
insert_datetime_mock()
module.create_archive(
excludes_filename='excludes',
verbosity=VERBOSITY_LOTS,
source_directories='foo bar',
repository='repo',
command='attic',
)
@@ -81,41 +95,53 @@ def test_make_prune_flags_should_return_flags_from_config():
assert tuple(result) == BASE_PRUNE_FLAGS
PRUNE_COMMAND = (
'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly', '3',
)
def test_prune_archives_should_call_attic_with_parameters():
retention_config = flexmock()
flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
BASE_PRUNE_FLAGS,
)
insert_subprocess_mock(
(
'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly',
'3',
),
)
insert_subprocess_mock(PRUNE_COMMAND)
module.prune_archives(
verbose=False,
verbosity=None,
repository='repo',
retention_config=retention_config,
command='attic',
)
def test_prune_archives_with_verbose_should_call_attic_with_verbose_parameters():
def test_prune_archives_with_verbosity_some_should_call_attic_with_stats_parameter():
retention_config = flexmock()
flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
BASE_PRUNE_FLAGS,
)
insert_subprocess_mock(
(
'attic', 'prune', 'repo', '--keep-daily', '1', '--keep-weekly', '2', '--keep-monthly',
'3', '--verbose',
),
)
insert_subprocess_mock(PRUNE_COMMAND + ('--stats',))
module.prune_archives(
repository='repo',
verbose=True,
verbosity=VERBOSITY_SOME,
retention_config=retention_config,
command='attic',
)
def test_prune_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter():
retention_config = flexmock()
flexmock(module).should_receive('_make_prune_flags').with_args(retention_config).and_return(
BASE_PRUNE_FLAGS,
)
insert_subprocess_mock(PRUNE_COMMAND + ('--verbose', '--stats',))
module.prune_archives(
repository='repo',
verbosity=VERBOSITY_LOTS,
retention_config=retention_config,
command='attic',
)
@@ -166,17 +192,18 @@ def test_check_archives_should_call_attic_with_parameters():
)
insert_platform_mock()
insert_datetime_mock()
flexmock(module).open = lambda filename, mode: stdout
flexmock(module).os = flexmock().should_receive('devnull').mock
builtins_mock().should_receive('open').and_return(stdout)
flexmock(module.os).should_receive('devnull')
module.check_archives(
verbose=False,
verbosity=None,
repository='repo',
consistency_config=consistency_config,
command='attic',
)
def test_check_archives_with_verbose_should_call_attic_with_verbose_parameters():
def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_parameter():
consistency_config = flexmock()
flexmock(module).should_receive('_parse_checks').and_return(flexmock())
flexmock(module).should_receive('_make_check_flags').and_return(())
@@ -188,9 +215,29 @@ def test_check_archives_with_verbose_should_call_attic_with_verbose_parameters()
insert_datetime_mock()
module.check_archives(
verbose=True,
verbosity=VERBOSITY_SOME,
repository='repo',
consistency_config=consistency_config,
command='attic',
)
def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter():
consistency_config = flexmock()
flexmock(module).should_receive('_parse_checks').and_return(flexmock())
flexmock(module).should_receive('_make_check_flags').and_return(())
insert_subprocess_mock(
('attic', 'check', 'repo', '--verbose'),
stdout=None,
)
insert_platform_mock()
insert_datetime_mock()
module.check_archives(
verbosity=VERBOSITY_LOTS,
repository='repo',
consistency_config=consistency_config,
command='attic',
)
@@ -200,9 +247,8 @@ def test_check_archives_without_any_checks_should_bail():
insert_subprocess_never()
module.check_archives(
verbose=False,
verbosity=None,
repository='repo',
consistency_config=consistency_config,
command='attic',
)
+33
View File
@@ -0,0 +1,33 @@
from flexmock import flexmock
from atticmatic import command as module
def test_load_backend_with_atticmatic_command_should_return_attic_backend():
backend = flexmock()
(
flexmock(module).should_receive('import_module').with_args('atticmatic.backends.attic')
.and_return(backend).once()
)
assert module.load_backend('atticmatic') == backend
def test_load_backend_with_unknown_command_should_return_attic_backend():
backend = flexmock()
(
flexmock(module).should_receive('import_module').with_args('atticmatic.backends.attic')
.and_return(backend).once()
)
assert module.load_backend('unknownmatic') == backend
def test_load_backend_with_borgmatic_command_should_return_borg_backend():
backend = flexmock()
(
flexmock(module).should_receive('import_module').with_args('atticmatic.backends.borg')
.and_return(backend).once()
)
assert module.load_backend('borgmatic') == backend
+2 -3
View File
@@ -197,9 +197,8 @@ def test_parse_section_options_for_missing_section_should_return_empty_dict():
def insert_mock_parser():
parser = flexmock()
parser.should_receive('readfp')
flexmock(module).open = lambda filename: None
flexmock(module).ConfigParser = parser
parser.should_receive('read')
module.ConfigParser = lambda: parser
return parser
+2
View File
@@ -0,0 +1,2 @@
VERBOSITY_SOME = 1
VERBOSITY_LOTS = 2
+1 -1
View File
@@ -2,7 +2,7 @@
# Space-separated list of source directories to backup.
source_directories: /home /etc
# Path to local or remote Attic repository.
# Path to local or remote repository.
repository: user@backupserver:sourcehostname.attic
[retention]
+8 -3
View File
@@ -2,12 +2,17 @@ from setuptools import setup, find_packages
setup(
name='atticmatic',
version='0.0.6',
description='A wrapper script for Attic backup software that creates and prunes backups',
version='0.1.0',
description='A wrapper script for Attic/Borg backup software that creates and prunes backups',
author='Dan Helfman',
author_email='witten@torsion.org',
packages=find_packages(),
entry_points={'console_scripts': ['atticmatic = atticmatic.command:main']},
entry_points={
'console_scripts': [
'atticmatic = atticmatic.command:main',
'borgmatic = atticmatic.command:main',
]
},
tests_require=(
'flexmock',
'nose',