Cutting a release.

Fix broken link to Borg quickstart

.gitignore

@@ -0,0 +1,6 @@
+*.egg-info
+*.pyc
+*.swp
+.tox
+build
+dist

.hgtags

@@ -21,3 +21,7 @@ acc7fb61566fe8028c179f43ecc735c851220b06 0.1.3
 e58246fc92bb22c2b2fd8b86a1227de69d2d0315 0.1.4
 0afff209b902698c2266986129d6dc9f5f913101 0.1.5
 4c63f3d90ec2bf6af1714a3acec84654a7c9edf3 0.1.6
+5a458ebef804be14e30d7375e3e9fbc26aedb80d 0.1.7
+977f19c2f6a515be6c5ef69cf17b0e0989532209 github/yaml_config_files
+0000000000000000000000000000000000000000 github/yaml_config_files
+28434dd0440cc8da44c2f3e9bd7e9402a59c3b40 github/master

AUTHORS

@@ -2,3 +2,4 @@ Dan Helfman <witten@torsion.org>: Main developer
 
 Alexander Görtz: Python 3 compatibility
 Henning Schroeder: Copy editing
+Robin `ypid` Schneider: Support additional options of Borg

NEWS

@@ -1,3 +1,12 @@
+0.1.8
+
+ * Fix for handling of spaces in source_directories which resulted in backup up everything.
+ * Fix for broken links to Borg documentation.
+ * At verbosity zero, suppressing Borg check stderr spew about "Checking segments".
+ * Support for Borg --one-file-system.
+ * Support for Borg create --umask.
+ * Support for file globs in source_directories.
+
 0.1.7
 
  * #11: Fixed parsing of punctuation in configuration file.

README.md

@@ -4,7 +4,7 @@ title: Atticmatic
 
 atticmatic is a simple Python wrapper script for the
 [Attic](https://attic-backup.org/) and
-[Borg](https://borgbackup.github.io/borgbackup/) backup software that
+[Borg](https://borgbackup.readthedocs.org/en/stable/) backup software that
 initiates a backup, prunes any old backups according to a retention policy,
 and validates backups for consistency. The script supports specifying your
 settings in a declarative configuration file rather than having to put them
@@ -12,22 +12,25 @@ all on the command-line, and handles common errors.
 
 Here's an example config file:
 
-    [location]
-    # Space-separated list of source directories to backup.
-    source_directories: /home /etc
+```INI
+[location]
+# Space-separated list of source directories to backup.
+# Globs are expanded.
+source_directories: /home /etc /var/log/syslog*
 
-    # Path to local or remote backup repository.
-    repository: user@backupserver:sourcehostname.attic
+# Path to local or remote backup repository.
+repository: user@backupserver:sourcehostname.attic
 
-    [retention]
-    # Retention policy for how many backups to keep in each category.
-    keep_daily: 7
-    keep_weekly: 4
-    keep_monthly: 6
+[retention]
+# Retention policy for how many backups to keep in each category.
+keep_daily: 7
+keep_weekly: 4
+keep_monthly: 6
 
-    [consistency]
-    # Consistency checks to run, or "disabled" to prevent checks.
-    checks: repository archives
+[consistency]
+# Consistency checks to run, or "disabled" to prevent checks.
+checks: repository archives
+```
 
 Additionally, exclude patterns can be specified in a separate excludes config
 file, one pattern per line.
@@ -42,7 +45,7 @@ available](https://torsion.org/hg/atticmatic). It's also mirrored on
 
 To get up and running, follow the [Attic Quick
 Start](https://attic-backup.org/quickstart.html) or the [Borg Quick
-Start](https://borgbackup.github.io/borgbackup/quickstart.html) to create a
+Start](https://borgbackup.readthedocs.org/en/latest/quickstart.html) to create a
 repository on a local or remote host. Note that if you plan to run atticmatic
 on a schedule with cron, and you encrypt your attic repository with a
 passphrase instead of a key file, you'll need to set the atticmatic
@@ -117,7 +120,7 @@ Then, to actually run tests, run:
 ### Broken pipe with remote repository
 
 When running atticmatic on a large remote repository, you may receive errors
-like the following, particularly while "attic check" is valiating backups for
+like the following, particularly while "attic check" is validating backups for
 consistency:
 
     Write failed: Broken pipe

atticmatic/backends/borg.py

@@ -7,12 +7,20 @@ from atticmatic.backends import shared
 
 COMMAND = 'borg'
 CONFIG_FORMAT = (
-    shared.CONFIG_FORMAT[0],  # location
+    Section_format(
+        'location',
+        (
+            option('source_directories'),
+            option('one_file_system', value_type=bool, required=False),
+            option('repository'),
+        ),
+    ),
     Section_format(
         'storage',
         (
             option('encryption_passphrase', required=False),
             option('compression', required=False),
+            option('umask', required=False),
         ),
     ),
     shared.CONFIG_FORMAT[2],  # retention
@@ -27,6 +35,7 @@ CONFIG_FORMAT = (
 
 
 initialize = partial(shared.initialize, command=COMMAND)
+
 create_archive = partial(shared.create_archive, command=COMMAND)
 prune_archives = partial(shared.prune_archives, command=COMMAND)
 check_archives = partial(shared.check_archives, command=COMMAND)

atticmatic/backends/shared.py

@@ -1,7 +1,10 @@
 from datetime import datetime
 import os
+import re
 import platform
 import subprocess
+from glob import glob
+from itertools import chain
 
 from atticmatic.config import Section_format, option
 from atticmatic.verbosity import VERBOSITY_SOME, VERBOSITY_LOTS
@@ -57,16 +60,21 @@ def initialize(storage_config, command):
 
 def create_archive(
     excludes_filename, verbosity, storage_config, source_directories, repository, command,
+    one_file_system=None
 ):
     '''
     Given an excludes filename (or None), a vebosity flag, a storage config dict, a space-separated
     list of source directories, a local or remote repository path, and a command to run, create an
     attic archive.
     '''
-    sources = tuple(source_directories.split(' '))
+    sources = re.split('\s+', source_directories)
+    sources = tuple(chain.from_iterable(glob(x) or [x] for x in sources))
     exclude_flags = ('--exclude-from', excludes_filename) if excludes_filename else ()
     compression = storage_config.get('compression', None)
     compression_flags = ('--compression', compression) if compression else ()
+    umask = storage_config.get('umask', None)
+    umask_flags = ('--umask', str(umask)) if umask else ()
+    one_file_system_flags = ('--one-file-system',) if one_file_system else ()
     verbosity_flags = {
         VERBOSITY_SOME: ('--stats',),
         VERBOSITY_LOTS: ('--verbose', '--stats'),
@@ -79,7 +87,8 @@ def create_archive(
             hostname=platform.node(),
             timestamp=datetime.now().isoformat(),
         ),
-    ) + sources + exclude_flags + compression_flags + verbosity_flags
+    ) + sources + exclude_flags + compression_flags + one_file_system_flags + \
+        umask_flags + verbosity_flags
 
     subprocess.check_call(full_command)
 
@@ -167,7 +176,7 @@ def _make_check_flags(checks, check_last=None):
         ('repository',)
 
     This will be returned as:
-    
+
         ('--repository-only',)
 
     Additionally, if a check_last value is given, a "--last" flag will be added. Note that only
@@ -204,7 +213,7 @@ def check_archives(verbosity, repository, consistency_config, command):
         repository,
     ) + _make_check_flags(checks, check_last) + verbosity_flags
 
-    # The check command spews to stdout even without the verbose flag. Suppress it.
+    # The check command spews to stdout/stderr even without the verbose flag. Suppress it.
     stdout = None if verbosity_flags else open(os.devnull, 'w')
 
-    subprocess.check_call(full_command, stdout=stdout)
+    subprocess.check_call(full_command, stdout=stdout, stderr=subprocess.STDOUT)

atticmatic/config.py

@@ -92,6 +92,7 @@ def parse_section_options(parser, section_format):
     type_getter = {
         str: parser.get,
         int: parser.getint,
+        bool: parser.getboolean,
     }
 
     return OrderedDict(

atticmatic/tests/integration/test_command.py

@@ -2,7 +2,7 @@ import os
 import sys
 
 from flexmock import flexmock
-from nose.tools import assert_raises
+import pytest
 
 from atticmatic import command as module
 
@@ -66,7 +66,7 @@ def test_parse_arguments_with_invalid_arguments_exits():
     sys.stderr = sys.stdout
 
     try:
-        with assert_raises(SystemExit):
+        with pytest.raises(SystemExit):
             module.parse_arguments(COMMAND_NAME, '--posix-me-harder')
     finally:
         sys.stderr = original_stderr

atticmatic/tests/unit/backends/test_shared.py

@@ -1,4 +1,5 @@
 from collections import OrderedDict
+from subprocess import STDOUT
 import os
 
 from flexmock import flexmock
@@ -31,7 +32,7 @@ def test_initialize_without_passphrase_should_not_set_environment():
 
 
 def insert_subprocess_mock(check_call_command, **kwargs):
-    subprocess = flexmock()
+    subprocess = flexmock(STDOUT=STDOUT)
     subprocess.should_receive('check_call').with_args(check_call_command, **kwargs).once()
     flexmock(module).subprocess = subprocess
 
@@ -71,6 +72,21 @@ def test_create_archive_should_call_attic_with_parameters():
     )
 
 
+def test_create_archive_with_two_spaces_in_source_directories():
+    insert_subprocess_mock(CREATE_COMMAND)
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        excludes_filename='excludes',
+        verbosity=None,
+        storage_config={},
+        source_directories='foo  bar',
+        repository='repo',
+        command='attic',
+    )
+
+
 def test_create_archive_with_none_excludes_filename_should_call_attic_without_excludes():
     insert_subprocess_mock(CREATE_COMMAND_WITHOUT_EXCLUDES)
     insert_platform_mock()
@@ -131,6 +147,85 @@ def test_create_archive_with_compression_should_call_attic_with_compression_para
     )
 
 
+def test_create_archive_with_one_file_system_should_call_attic_with_one_file_system_parameters():
+    insert_subprocess_mock(CREATE_COMMAND + ('--one-file-system',))
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        excludes_filename='excludes',
+        verbosity=None,
+        storage_config={},
+        source_directories='foo bar',
+        repository='repo',
+        command='attic',
+        one_file_system=True,
+    )
+
+
+def test_create_archive_with_umask_should_call_attic_with_umask_parameters():
+    insert_subprocess_mock(CREATE_COMMAND + ('--umask', '740'))
+    insert_platform_mock()
+    insert_datetime_mock()
+
+    module.create_archive(
+        excludes_filename='excludes',
+        verbosity=None,
+        storage_config={'umask': 740},
+        source_directories='foo bar',
+        repository='repo',
+        command='attic',
+    )
+
+
+def test_create_archive_with_source_directories_glob_expands():
+    insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'foo', 'food'))
+    insert_platform_mock()
+    insert_datetime_mock()
+    flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
+
+    module.create_archive(
+        excludes_filename=None,
+        verbosity=None,
+        storage_config={},
+        source_directories='foo*',
+        repository='repo',
+        command='attic',
+    )
+
+
+def test_create_archive_with_non_matching_source_directories_glob_passes_through():
+    insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'foo*'))
+    insert_platform_mock()
+    insert_datetime_mock()
+    flexmock(module).should_receive('glob').with_args('foo*').and_return([])
+
+    module.create_archive(
+        excludes_filename=None,
+        verbosity=None,
+        storage_config={},
+        source_directories='foo*',
+        repository='repo',
+        command='attic',
+    )
+
+
+def test_create_archive_with_glob_should_call_attic_with_expanded_directories():
+    insert_subprocess_mock(('attic', 'create', 'repo::host-now', 'foo', 'food'))
+    insert_platform_mock()
+    insert_datetime_mock()
+    flexmock(module).should_receive('glob').with_args('foo*').and_return(['foo', 'food'])
+
+    module.create_archive(
+        excludes_filename=None,
+        verbosity=None,
+        storage_config={},
+        source_directories='foo*',
+        repository='repo',
+        command='attic',
+    )
+
+
 BASE_PRUNE_FLAGS = (
     ('--keep-daily', '1'),
     ('--keep-weekly', '2'),
@@ -259,7 +354,7 @@ def test_check_archives_should_call_attic_with_parameters():
     stdout = flexmock()
     insert_subprocess_mock(
         ('attic', 'check', 'repo'),
-        stdout=stdout,
+        stdout=stdout, stderr=STDOUT,
     )
     insert_platform_mock()
     insert_datetime_mock()
@@ -280,7 +375,7 @@ def test_check_archives_with_verbosity_some_should_call_attic_with_verbose_param
     flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_subprocess_mock(
         ('attic', 'check', 'repo', '--verbose'),
-        stdout=None,
+        stdout=None, stderr=STDOUT,
     )
     insert_platform_mock()
     insert_datetime_mock()
@@ -299,7 +394,7 @@ def test_check_archives_with_verbosity_lots_should_call_attic_with_verbose_param
     flexmock(module).should_receive('_make_check_flags').and_return(())
     insert_subprocess_mock(
         ('attic', 'check', 'repo', '--verbose'),
-        stdout=None,
+        stdout=None, stderr=STDOUT,
     )
     insert_platform_mock()
     insert_datetime_mock()

atticmatic/tests/unit/test_config.py

@@ -1,7 +1,7 @@
 from collections import OrderedDict
 
 from flexmock import flexmock
-from nose.tools import assert_raises
+import pytest
 
 from atticmatic import config as module
 
@@ -61,7 +61,7 @@ def test_validate_configuration_format_with_missing_required_section_should_rais
         ),
     )
 
-    with assert_raises(ValueError):
+    with pytest.raises(ValueError):
         module.validate_configuration_format(parser, config_format)
 
 
@@ -96,7 +96,7 @@ def test_validate_configuration_format_with_unknown_section_should_raise():
         module.Section_format('section', options=()),
     )
 
-    with assert_raises(ValueError):
+    with pytest.raises(ValueError):
         module.validate_configuration_format(parser, config_format)
 
 
@@ -114,7 +114,7 @@ def test_validate_configuration_format_with_missing_required_option_should_raise
         ),
     )
 
-    with assert_raises(ValueError):
+    with pytest.raises(ValueError):
         module.validate_configuration_format(parser, config_format)
 
 
@@ -146,7 +146,7 @@ def test_validate_configuration_format_with_extra_option_should_raise():
         ),
     )
 
-    with assert_raises(ValueError):
+    with pytest.raises(ValueError):
         module.validate_configuration_format(parser, config_format)
 
 
@@ -154,6 +154,7 @@ def test_parse_section_options_should_return_section_options():
     parser = flexmock()
     parser.should_receive('get').with_args('section', 'foo').and_return('value')
     parser.should_receive('getint').with_args('section', 'bar').and_return(1)
+    parser.should_receive('getboolean').never()
     parser.should_receive('has_option').with_args('section', 'foo').and_return(True)
     parser.should_receive('has_option').with_args('section', 'bar').and_return(True)
 
@@ -179,6 +180,7 @@ def test_parse_section_options_for_missing_section_should_return_empty_dict():
     parser = flexmock()
     parser.should_receive('get').never()
     parser.should_receive('getint').never()
+    parser.should_receive('getboolean').never()
     parser.should_receive('has_option').with_args('section', 'foo').and_return(False)
     parser.should_receive('has_option').with_args('section', 'bar').and_return(False)
 
@@ -226,5 +228,5 @@ def test_parse_configuration_with_file_open_error_should_raise():
     parser = insert_mock_parser()
     parser.should_receive('read').and_return([])
 
-    with assert_raises(ValueError):
+    with pytest.raises(ValueError):
         module.parse_configuration('filename', config_format=flexmock())

sample/config

@@ -1,6 +1,11 @@
 [location]
 # Space-separated list of source directories to backup.
-source_directories: /home /etc
+# Globs are expanded.
+source_directories: /home /etc /var/log/syslog*
+
+# For Borg only, you can specify to stay in same file system (do not cross
+# mount points).
+#one_file_system: True
 
 # Path to local or remote repository.
 repository: user@backupserver:sourcehostname.attic
@@ -10,14 +15,16 @@ repository: user@backupserver:sourcehostname.attic
 # were initialized with passphrase/repokey encryption.
 #encryption_passphrase: foo
 # For Borg only, you can specify the type of compression to use when creating
-# archives. See https://borgbackup.github.io/borgbackup/usage.html#borg-create
+# archives. See https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create
 # for details. Defaults to no compression.
 #compression: lz4
+# For Borg only, you can specify the umask to be used for borg create.
+#umask: 0740
 
 [retention]
 # Retention policy for how many backups to keep in each category. See
 # https://attic-backup.org/usage.html#attic-prune or
-# https://borgbackup.github.io/borgbackup/usage.html#borg-prune for details.
+# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-prune for details.
 #keep_within: 3H
 #keep_hourly: 24
 keep_daily: 7
@@ -30,7 +37,7 @@ keep_yearly: 1
 # Space-separated list of consistency checks to run: "repository", "archives",
 # or both. Defaults to both. Set to "disabled" to disable all consistency
 # checks. See https://attic-backup.org/usage.html#attic-check or
-# https://borgbackup.github.io/borgbackup/usage.html#borg-check for details.
+# https://borgbackup.readthedocs.org/en/stable/usage.html#borg-check for details.
 checks: repository archives
 # For Borg only, you can restrict the number of checked archives to the last n.
 #check_last: 3

setup.cfg

@@ -1,5 +1,2 @@
 [metadata]
 description-file=README.md
-
-[nosetests]
-detailed-errors=1

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '0.1.7'
+VERSION = '0.1.8'
 
 
 setup(
@@ -30,6 +30,6 @@ setup(
     },
     tests_require=(
         'flexmock',
-        'nose',
+        'pytest',
     )
 )

test_requirements.txt

@@ -1,2 +1,2 @@
-flexmock==0.9.7
-nose==1.3.4
+flexmock==0.10.2
+pytest==2.9.1

tox.ini

@@ -5,4 +5,4 @@ skipsdist=True
 [testenv]
 usedevelop=True
 deps=-rtest_requirements.txt
-commands = nosetests []
+commands = py.test []