#1: Add support for "borg check --last N" to Borg backend.

.hgtags

@@ -10,3 +10,8 @@ aa8a807f4ba28f0652764ed14713ffea2fd6922d 0.0.5
 a03495a8e8b471da63b5e2ae79d3ff9065839c2a 0.0.5
 7ea93ca83f426ec0a608a68580c72c0775b81f86 0.0.6
 cf4c7065f0711deda1cba878398bc05390e2c3f9 0.0.7
+38d72677343f0a5d6845f4ac50d6778397083d45 0.1.0
+ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1
+ac5dfa01e9d14d09845f5e94c2c679e21c5eb2f9 0.1.1
+7b6c87dca7ea312b2257ac1b46857b3f8c56b39c 0.1.1
+83067f995dd391e38544a7722dc3b254b59c5521 0.1.2

NEWS

@@ -1,3 +1,16 @@
+0.1.3
+
+ * #1: Add support for "borg check --last N" to Borg backend.
+
+0.1.2
+
+ * As a convenience to new users, allow a missing default excludes file.
+ * New issue tracker, linked from documentation.
+
+0.1.1
+
+ * Adding borgmatic cron example, and updating documentation to refer to it.
+
 0.1.0
 
  * New "borgmatic" command to support Borg backup software, a fork of Attic.

README.md

@@ -1,6 +1,4 @@
 title: Atticmatic
-date:
-save_as: atticmatic/index.html
 
 ## Overview
 
@@ -66,7 +64,7 @@ If you are using Attic, copy the following configuration files:
 
 If you are using Borg, copy the files like this instead:
 
-    sudo cp sample/atticmatic.cron /etc/cron.d/borgmatic
+    sudo cp sample/borgmatic.cron /etc/cron.d/borgmatic
     sudo mkdir /etc/borgmatic/
     sudo cp sample/config sample/excludes /etc/borgmatic/
 
@@ -135,6 +133,9 @@ This should make the client keep the connection alive while validating
 backups.
 
 
-## Feedback
+## Issues and feedback
 
-Questions? Comments? Got a patch? Contact <mailto:witten@torsion.org>.
+Got an issue or an idea for a feature enhancement? Check out the [atticmatic
+issue tracker](https://tree.taiga.io/project/witten-atticmatic/issues).
+
+Other questions or comments? Contact <mailto:witten@torsion.org>.

atticmatic/backends/attic.py

@@ -5,7 +5,7 @@ from atticmatic.backends import shared
 # An atticmatic backend that supports Attic for actually handling backups.
 
 COMMAND = 'attic'
-
+CONFIG_FORMAT = shared.CONFIG_FORMAT
 
 create_archive = partial(shared.create_archive, command=COMMAND)
 prune_archives = partial(shared.prune_archives, command=COMMAND)

atticmatic/backends/borg.py

@@ -1,13 +1,24 @@
 from functools import partial
 
+from atticmatic.config import Section_format, option
 from atticmatic.backends import shared
 
 # An atticmatic backend that supports Borg for actually handling backups.
 
 COMMAND = 'borg'
+CONFIG_FORMAT = (
+    shared.CONFIG_FORMAT[0],  # location
+    shared.CONFIG_FORMAT[1],  # retention
+    Section_format(
+        'consistency',
+        (
+            option('checks', required=False),
+            option('check_last', required=False),
+        ),
+    )
+)
 
 
 create_archive = partial(shared.create_archive, command=COMMAND)
 prune_archives = partial(shared.prune_archives, command=COMMAND)
 check_archives = partial(shared.check_archives, command=COMMAND)
-

atticmatic/backends/shared.py

@@ -3,6 +3,7 @@ import os
 import platform
 import subprocess
 
+from atticmatic.config import Section_format, option
 from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 
 
@@ -12,12 +13,41 @@ from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
 # atticmatic.backends.borg.
 
 
+CONFIG_FORMAT = (
+    Section_format(
+        'location',
+        (
+            option('source_directories'),
+            option('repository'),
+        ),
+    ),
+    Section_format(
+        'retention',
+        (
+            option('keep_within', required=False),
+            option('keep_hourly', int, required=False),
+            option('keep_daily', int, required=False),
+            option('keep_weekly', int, required=False),
+            option('keep_monthly', int, required=False),
+            option('keep_yearly', int, required=False),
+            option('prefix', required=False),
+        ),
+    ),
+    Section_format(
+        'consistency',
+        (
+            option('checks', required=False),
+        ),
+    )
+)
+
 def create_archive(excludes_filename, verbosity, source_directories, repository, command):
     '''
-    Given an excludes filename, a vebosity flag, a space-separated list of source directories, a
+    Given an excludes filename (or None), a vebosity flag, a space-separated list of source
-    local or remote repository path, and a command to run, create an attic archive.
+    directories, a local or remote repository path, and a command to run, create an attic archive.
     '''
     sources = tuple(source_directories.split(' '))
+    exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else ()
     verbosity_flags = {
         VERBOSITY_SOME: ('--stats',),
         VERBOSITY_LOTS: ('--verbose', '--stats'),
@@ -25,13 +55,12 @@ def create_archive(excludes_filename, verbosity, source_directories, repository,
 
     full_command = (
         command, 'create',
-        '--exclude-from', excludes_filename,
         '{repo}::{hostname}-{timestamp}'.format(
             repo=repository,
             hostname=platform.node(),
             timestamp=datetime.now().isoformat(),
         ),
-    ) + sources + verbosity_flags
+    ) + sources + exclude_flags + verbosity_flags
 
     subprocess.check_call(full_command)
 
@@ -110,7 +139,7 @@ def _parse_checks(consistency_config):
     )
 
 
-def _make_check_flags(checks):
+def _make_check_flags(checks, check_last=None):
     '''
     Given a parsed sequence of checks, transform it into tuple of command-line flags.
 
@@ -121,13 +150,17 @@ def _make_check_flags(checks):
     This will be returned as:
     
         ('--repository-only',)
+
+    Additionally, if a check_last value is given, a "--last" flag will be added. Note that only
+    Borg supports this flag.
     '''
+    last_flag = ('--last', check_last) if check_last else ()
     if checks == DEFAULT_CHECKS:
-        return ()
+        return last_flag
 
     return tuple(
         '--{}-only'.format(check) for check in checks
-    )
+    ) + last_flag
 
 
 def check_archives(verbosity, repository, consistency_config, command):
@@ -138,6 +171,7 @@ def check_archives(verbosity, repository, consistency_config, command):
     If there are no consistency checks to run, skip running them.
     '''
     checks = _parse_checks(consistency_config)
+    check_last = consistency_config.get('check_last', None)
     if not checks:
         return
 
@@ -149,7 +183,7 @@ def check_archives(verbosity, repository, consistency_config, command):
     full_command = (
         command, 'check',
         repository,
-    ) + _make_check_flags(checks) + verbosity_flags
+    ) + _make_check_flags(checks, check_last) + verbosity_flags
 
     # The check command spews to stdout even without the verbose flag. Suppress it.
     stdout = None if verbosity_flags else open(os.devnull, 'w')

atticmatic/command.py

@@ -18,17 +18,20 @@ def parse_arguments(command_name, *arguments):
     parse the arguments and return them as an ArgumentParser instance. Use the command name to
     determine the default configuration and excludes paths.
     '''
+    config_filename_default = DEFAULT_CONFIG_FILENAME_PATTERN.format(command_name)
+    excludes_filename_default = DEFAULT_EXCLUDES_FILENAME_PATTERN.format(command_name)
+
     parser = ArgumentParser()
     parser.add_argument(
         '-c', '--config',
         dest='config_filename',
-        default=DEFAULT_CONFIG_FILENAME_PATTERN.format(command_name),
+        default=config_filename_default,
         help='Configuration filename',
     )
     parser.add_argument(
         '--excludes',
         dest='excludes_filename',
-        default=DEFAULT_EXCLUDES_FILENAME_PATTERN.format(command_name),
+        default=excludes_filename_default if os.path.exists(excludes_filename_default) else None,
         help='Excludes filename',
     )
     parser.add_argument(
@@ -57,9 +60,9 @@ def main():
     try:
         command_name = os.path.basename(sys.argv[0])
         args = parse_arguments(command_name, *sys.argv[1:])
-        config = parse_configuration(args.config_filename)
-        repository = config.location['repository']
         backend = load_backend(command_name)
+        config = parse_configuration(args.config_filename, backend.CONFIG_FORMAT)
+        repository = config.location['repository']
 
         backend.create_archive(args.excludes_filename, args.verbosity, **config.location)
         backend.prune_archives(args.verbosity, repository, config.retention)

atticmatic/config.py

@@ -20,35 +20,6 @@ def option(name, value_type=str, required=True):
     return Config_option(name, value_type, required)
 
 
-CONFIG_FORMAT = (
-    Section_format(
-        'location',
-        (
-            option('source_directories'),
-            option('repository'),
-        ),
-    ),
-    Section_format(
-        'retention',
-        (
-            option('keep_within', required=False),
-            option('keep_hourly', int, required=False),
-            option('keep_daily', int, required=False),
-            option('keep_weekly', int, required=False),
-            option('keep_monthly', int, required=False),
-            option('keep_yearly', int, required=False),
-            option('prefix', required=False),
-        ),
-    ),
-    Section_format(
-        'consistency',
-        (
-            option('checks', required=False),
-        ),
-    )
-)
-
-
 def validate_configuration_format(parser, config_format):
     '''
     Given an open ConfigParser and an expected config file format, validate that the parsed
@@ -110,11 +81,6 @@ def validate_configuration_format(parser, config_format):
             )
 
 
-# Describes a parsed configuration, where each attribute is the name of a configuration file section 
-# and each value is a dict of that section's parsed options.
-Parsed_config = namedtuple('Config', (section_format.name for section_format in CONFIG_FORMAT))
-
-
 def parse_section_options(parser, section_format):
     '''
     Given an open ConfigParser and an expected section format, return the option values from that
@@ -135,21 +101,25 @@ def parse_section_options(parser, section_format):
     )
 
 
-def parse_configuration(config_filename):
+def parse_configuration(config_filename, config_format):
     '''
-    Given a config filename of the expected format, return the parsed configuration as Parsed_config
+    Given a config filename and an expected config file format, return the parsed configuration
-    data structure.
+    as a namedtuple with one attribute for each parsed section.
 
     Raise IOError if the file cannot be read, or ValueError if the format is not as expected.
     '''
     parser = ConfigParser()
     parser.read(config_filename)
 
-    validate_configuration_format(parser, CONFIG_FORMAT)
+    validate_configuration_format(parser, config_format)
+
+    # Describes a parsed configuration, where each attribute is the name of a configuration file
+    # section and each value is a dict of that section's parsed options.
+    Parsed_config = namedtuple('Parsed_config', (section_format.name for section_format in config_format))
 
     return Parsed_config(
         *(
             parse_section_options(parser, section_format)
-            for section_format in CONFIG_FORMAT
+            for section_format in config_format
         )
     )

atticmatic/tests/integration/test_command.py

@@ -1,5 +1,7 @@
+import os
 import sys
 
+from flexmock import flexmock
 from nose.tools import assert_raises
 
 from atticmatic import command as module
@@ -9,6 +11,8 @@ COMMAND_NAME = 'foomatic'
 
 
 def test_parse_arguments_with_no_arguments_uses_defaults():
+    flexmock(os.path).should_receive('exists').and_return(True)
+
     parser = module.parse_arguments(COMMAND_NAME)
 
     assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME)
@@ -17,6 +21,8 @@ def test_parse_arguments_with_no_arguments_uses_defaults():
 
 
 def test_parse_arguments_with_filename_arguments_overrides_defaults():
+    flexmock(os.path).should_receive('exists').and_return(True)
+
     parser = module.parse_arguments(COMMAND_NAME, '--config', 'myconfig', '--excludes', 'myexcludes')
 
     assert parser.config_filename == 'myconfig'
@@ -24,7 +30,29 @@ def test_parse_arguments_with_filename_arguments_overrides_defaults():
     assert parser.verbosity == None
 
 
+def test_parse_arguments_with_missing_default_excludes_file_sets_filename_to_none():
+    flexmock(os.path).should_receive('exists').and_return(False)
+
+    parser = module.parse_arguments(COMMAND_NAME)
+
+    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME)
+    assert parser.excludes_filename == None
+    assert parser.verbosity == None
+
+
+def test_parse_arguments_with_missing_overridden_excludes_file_retains_filename():
+    flexmock(os.path).should_receive('exists').and_return(False)
+
+    parser = module.parse_arguments(COMMAND_NAME, '--excludes', 'myexcludes')
+
+    assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME)
+    assert parser.excludes_filename == 'myexcludes'
+    assert parser.verbosity == None
+
+
 def test_parse_arguments_with_verbosity_flag_overrides_default():
+    flexmock(os.path).should_receive('exists').and_return(True)
+
     parser = module.parse_arguments(COMMAND_NAME, '--verbosity', '1')
 
     assert parser.config_filename == module.DEFAULT_CONFIG_FILENAME_PATTERN.format(COMMAND_NAME)
@@ -33,6 +61,7 @@ def test_parse_arguments_with_verbosity_flag_overrides_default():
 
 
 def test_parse_arguments_with_invalid_arguments_exits():
+    flexmock(os.path).should_receive('exists').and_return(True)
     original_stderr = sys.stderr
     sys.stderr = sys.stdout
 

atticmatic/tests/unit/backends/test_shared.py

@@ -29,7 +29,8 @@ def insert_datetime_mock():
     ).mock
 
 
-CREATE_COMMAND = ('attic', 'create', '--exclude-from', 'excludes', 'repo::host-now', 'foo', 'bar')
+CREATE_COMMAND_WITHOUT_EXCLUDES = ('attic', 'create', 'repo::host-now', 'foo', 'bar')
+CREATE_COMMAND = CREATE_COMMAND_WITHOUT_EXCLUDES + ('--exclude-from', 'excludes')
 
 
 def test_create_archive_should_call_attic_with_parameters():
@@ -46,6 +47,20 @@ def test_create_archive_should_call_attic_with_parameters():
     )
 
 
+def test_create_archive_with_none_excludes_filename_should_call_attic_without_excludes():
+    insert_subprocess_mock(CREATE_COMMAND_WITHOUT_EXCLUDES)
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        excludes_filename=None,
+        verbosity=None,
+        source_directories='foo bar',
+        repository='repo',
+        command='attic',
+    )
+
+
 def test_create_archive_with_verbosity_some_should_call_attic_with_stats_parameter():
     insert_subprocess_mock(CREATE_COMMAND + ('--stats',))
     insert_platform_mock()
@@ -181,10 +196,24 @@ def test_make_check_flags_with_default_checks_returns_no_flags():
     assert flags == ()
 
 
+def test_make_check_flags_with_checks_and_last_returns_flags_including_last():
+    flags = module._make_check_flags(('foo', 'bar'), check_last=3)
+
+    assert flags == ('--foo-only', '--bar-only', '--last', 3)
+
+
+def test_make_check_flags_with_last_returns_last_flag():
+    flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3)
+
+    assert flags == ('--last', 3)
+
+
 def test_check_archives_should_call_attic_with_parameters():
-    consistency_config = flexmock()
+    checks = flexmock()
-    flexmock(module).should_receive('_parse_checks').and_return(flexmock())
+    check_last = flexmock()
-    flexmock(module).should_receive('_make_check_flags').and_return(())
+    consistency_config = flexmock().should_receive('get').and_return(check_last).mock
+    flexmock(module).should_receive('_parse_checks').and_return(checks)
+    flexmock(module).should_receive('_make_check_flags').with_args(checks, check_last).and_return(())
     stdout = flexmock()
     insert_subprocess_mock(
         ('attic', 'check', 'repo'),
@@ -204,7 +233,7 @@ def test_check_archives_should_call_attic_with_parameters():
 
 
 def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_parameter():
-    consistency_config = flexmock()
+    consistency_config = flexmock().should_receive('get').and_return(None).mock
     flexmock(module).should_receive('_parse_checks').and_return(flexmock())
     flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_subprocess_mock(
@@ -223,7 +252,7 @@ def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_param
 
 
 def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_parameter():
-    consistency_config = flexmock()
+    consistency_config = flexmock().should_receive('get').and_return(None).mock
     flexmock(module).should_receive('_parse_checks').and_return(flexmock())
     flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_subprocess_mock(
@@ -242,7 +271,7 @@ def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_param
 
 
 def test_check_archives_without_any_checks_should_bail():
-    consistency_config = flexmock()
+    consistency_config = flexmock().should_receive('get').and_return(None).mock
     flexmock(module).should_receive('_parse_checks').and_return(())
     insert_subprocess_never()
 

atticmatic/tests/unit/test_config.py

@@ -205,17 +205,18 @@ def insert_mock_parser():
 
 def test_parse_configuration_should_return_section_configs():
     parser = insert_mock_parser()
+    config_format = (flexmock(name='items'), flexmock(name='things'))
     mock_module = flexmock(module)
     mock_module.should_receive('validate_configuration_format').with_args(
-        parser, module.CONFIG_FORMAT,
+        parser, config_format,
     ).once()
-    mock_section_configs = (flexmock(),) * len(module.CONFIG_FORMAT)
+    mock_section_configs = (flexmock(), flexmock())
 
-    for section_format, section_config in zip(module.CONFIG_FORMAT, mock_section_configs):
+    for section_format, section_config in zip(config_format, mock_section_configs):
         mock_module.should_receive('parse_section_options').with_args(
             parser, section_format,
         ).and_return(section_config).once()
 
-    parsed_config = module.parse_configuration('filename')
+    parsed_config = module.parse_configuration('filename', config_format)
 
-    assert parsed_config == module.Parsed_config(*mock_section_configs)
+    assert parsed_config == type(parsed_config)(*mock_section_configs)

sample/borgmatic.cron

@@ -0,0 +1,3 @@
+# You can drop this file into /etc/cron.d/ to run borgmatic nightly.
+
+0 3 * * * root PATH=$PATH:/usr/local/bin /usr/local/bin/borgmatic

sample/config

@@ -7,8 +7,9 @@ repository: user@backupserver:sourcehostname.attic
 
 [retention]
 # Retention policy for how many backups to keep in each category. See
-# https://attic-backup.org/usage.html#attic-prune for details.
+# https://attic-backup.org/usage.html#attic-prune or
-#keep_within: 3h
+# https://borgbackup.github.io/borgbackup/usage.html#borg-prune for details.
+#keep_within: 3H
 #keep_hourly: 24
 keep_daily: 7
 keep_weekly: 4
@@ -19,5 +20,8 @@ keep_yearly: 1
 [consistency]
 # Space-separated list of consistency checks to run: "repository", "archives",
 # or both. Defaults to both. Set to "disabled" to disable all consistency
-# checks. See https://attic-backup.org/usage.html#attic-check for details.
+# checks. See https://attic-backup.org/usage.html#attic-check or
+# https://borgbackup.github.io/borgbackup/usage.html#borg-check for details.
 checks: repository archives
+# For Borg only, you can restrict the number of checked archives to the last n.
+#check_last: 3

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='atticmatic',
-    version='0.1.0',
+    version='0.1.1',
     description='A wrapper script for Attic/Borg backup software that creates and prunes backups',
     author='Dan Helfman',
     author_email='witten@torsion.org',