Do not treat Borg warnings (exit code 1) as failures (#204).

Improved description of what the resource provides.

NEWS

@@ -1,3 +1,30 @@
+1.3.14
+ * #204: Do not treat Borg warnings (exit code 1) as failures.
+ * When validating configuration files, require strings instead of allowing any scalar type.
+
+1.3.13
+ * #199: Add note to documentation about using spaces instead of tabs for indentation, as YAML does
+   not allow tabs.
+ * #203: Fix compatibility with ruamel.yaml 0.16.x.
+ * If a "prefix" option in borgmatic's configuration has an empty value (blank or ""), then disable
+   default prefix.
+
+1.3.12
+ * Only log to syslog when run from a non-interactive console (e.g. a cron job).
+ * Remove unicode byte order mark from syslog output so it doesn't show up as a literal in rsyslog
+   output. See discussion on #197.
+
+1.3.11
+ * #193: Pass through several "borg list" and "borg info" flags like --short, --format, --sort-by,
+   --first, --last, etc. via borgmatic command-line flags.
+ * Add borgmatic info --repository and --archive command-line flags to display info for individual
+   repositories or archives.
+ * Support for Borg --noatime, --noctime, and --nobirthtime flags via corresponding options in
+   borgmatic configuration location section.
+
+1.3.10
+ * #198: Fix for Borg create error output not showing up at borgmatic verbosity level zero.
+
 1.3.9
  * #195: Switch to command-line actions as more traditional sub-commands, e.g. "borgmatic create",
    "borgmatic prune", etc. However, the classic dashed options like "--create" still work!

README.md

@@ -2,7 +2,7 @@
 title: borgmatic
 permalink: index.html
 ---
-<a href="https://build.torsion.org/witten/borgmatic" alt="build status">![Build Status](https://build.torsion.org/api/badges/witten/borgmatic/status.svg)</a>
+<a href="https://build.torsion.org/witten/borgmatic" alt="build status">![Build Status](https://build.torsion.org/api/badges/witten/borgmatic/status.svg?ref=refs/heads/master)</a>
 
 ## Overview
 
@@ -64,7 +64,7 @@ href="https://asciinema.org/a/203761" target="_blank">screencast</a>.
  * [Deal with very large backups](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/)
  * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/)
  * [Restore a backup](https://torsion.org/borgmatic/docs/how-to/restore-a-backup/)
- * [Run preparation steps before backups](https://torsion.org/borgmatic/docs/how-to/run-preparation-steps-before-backups/)
+ * [Add preparation and cleanup steps to backups](https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/)
  * [Upgrade borgmatic](https://torsion.org/borgmatic/docs/how-to/upgrade/)
  * [Develop on borgmatic](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/)
 

borgmatic/borg/check.py

@@ -55,7 +55,7 @@ def _make_check_flags(checks, check_last=None, prefix=None):
     '''
     if 'archives' in checks:
         last_flags = ('--last', str(check_last)) if check_last else ()
-        prefix_flags = ('--prefix', prefix) if prefix else ('--prefix', DEFAULT_PREFIX)
+        prefix_flags = ('--prefix', prefix) if prefix else ()
     else:
         last_flags = ()
         prefix_flags = ()
@@ -102,7 +102,7 @@ def check_archives(
         if logger.isEnabledFor(logging.DEBUG):
             verbosity_flags = ('--debug', '--show-rc')
 
-        prefix = consistency_config.get('prefix')
+        prefix = consistency_config.get('prefix', DEFAULT_PREFIX)
 
         full_command = (
             (local_path, 'check', repository)

borgmatic/borg/create.py

@@ -142,6 +142,9 @@ def create_archive(
         + (('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ())
         + (('--one-file-system',) if location_config.get('one_file_system') else ())
         + (('--numeric-owner',) if location_config.get('numeric_owner') else ())
+        + (('--noatime',) if location_config.get('atime') is False else ())
+        + (('--noctime',) if location_config.get('ctime') is False else ())
+        + (('--nobirthtime',) if location_config.get('birthtime') is False else ())
         + (('--read-special',) if location_config.get('read_special') else ())
         + (('--nobsdflags',) if location_config.get('bsd_flags') is False else ())
         + (('--files-cache', files_cache) if files_cache else ())

borgmatic/borg/flags.py

@@ -0,0 +1,31 @@
+import itertools
+
+
+def make_flags(name, value):
+    '''
+    Given a flag name and its value, return it formatted as Borg-compatible flags.
+    '''
+    if not value:
+        return ()
+
+    flag = '--{}'.format(name.replace('_', '-'))
+
+    if value is True:
+        return (flag,)
+
+    return (flag, str(value))
+
+
+def make_flags_from_arguments(arguments, excludes=()):
+    '''
+    Given borgmatic command-line arguments as an instance of argparse.Namespace, and optionally a
+    list of named arguments to exclude, generate and return the corresponding Borg command-line
+    flags as a tuple.
+    '''
+    return tuple(
+        itertools.chain.from_iterable(
+            make_flags(name, value=getattr(arguments, name))
+            for name in sorted(vars(arguments))
+            if name not in excludes and not name.startswith('_')
+        )
+    )

borgmatic/borg/info.py

@@ -1,26 +1,44 @@
 import logging
 
+from borgmatic.borg.flags import make_flags, make_flags_from_arguments
 from borgmatic.execute import execute_command
 
 logger = logging.getLogger(__name__)
 
 
 def display_archives_info(
-    repository, storage_config, local_path='borg', remote_path=None, json=False
+    repository, storage_config, info_arguments, local_path='borg', remote_path=None
 ):
     '''
-    Given a local or remote repository path, and a storage config dict, display summary information
-    for Borg archives in the repository or return JSON summary information.
+    Given a local or remote repository path, a storage config dict, and the arguments to the info
+    action, display summary information for Borg archives in the repository or return JSON summary
+    information.
     '''
     lock_wait = storage_config.get('lock_wait', None)
 
     full_command = (
-        (local_path, 'info', repository)
-        + (('--remote-path', remote_path) if remote_path else ())
-        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ())
-        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else ())
-        + (('--json',) if json else ())
+        (
+            local_path,
+            'info',
+            '::'.join((repository, info_arguments.archive))
+            if info_arguments.archive
+            else repository,
+        )
+        + (
+            ('--info',)
+            if logger.getEffectiveLevel() == logging.INFO and not info_arguments.json
+            else ()
+        )
+        + (
+            ('--debug', '--show-rc')
+            if logger.isEnabledFor(logging.DEBUG) and not info_arguments.json
+            else ()
+        )
+        + make_flags('remote-path', remote_path)
+        + make_flags('lock-wait', lock_wait)
+        + make_flags_from_arguments(info_arguments, excludes=('repository', 'archive'))
     )
 
-    return execute_command(full_command, output_log_level=None if json else logging.WARNING)
+    return execute_command(
+        full_command, output_log_level=None if info_arguments.json else logging.WARNING
+    )

borgmatic/borg/init.py

@@ -1,7 +1,7 @@
 import logging
 import subprocess
 
-from borgmatic.execute import execute_command
+from borgmatic.execute import BORG_ERROR_EXIT_CODE, execute_command
 
 logger = logging.getLogger(__name__)
 
@@ -44,4 +44,8 @@ def initialize_repository(
     )
 
     # Don't use execute_command() here because it doesn't support interactive prompts.
-    subprocess.check_call(init_command)
+    try:
+        subprocess.check_call(init_command)
+    except subprocess.CalledProcessError as error:
+        if error.returncode >= BORG_ERROR_EXIT_CODE:
+            raise

borgmatic/borg/list.py

@@ -1,27 +1,42 @@
 import logging
 
+from borgmatic.borg.flags import make_flags, make_flags_from_arguments
 from borgmatic.execute import execute_command
 
 logger = logging.getLogger(__name__)
 
 
-def list_archives(
-    repository, storage_config, archive=None, local_path='borg', remote_path=None, json=False
-):
+def list_archives(repository, storage_config, list_arguments, local_path='borg', remote_path=None):
     '''
-    Given a local or remote repository path and a storage config dict, display the output of listing
-    Borg archives in the repository or return JSON output. Or, if an archive name is given, listing
-    the files in that archive.
+    Given a local or remote repository path, a storage config dict, and the arguments to the list
+    action, display the output of listing Borg archives in the repository or return JSON output. Or,
+    if an archive name is given, listing the files in that archive.
     '''
     lock_wait = storage_config.get('lock_wait', None)
 
     full_command = (
-        (local_path, 'list', '::'.join((repository, archive)) if archive else repository)
-        + (('--remote-path', remote_path) if remote_path else ())
-        + (('--lock-wait', str(lock_wait)) if lock_wait else ())
-        + (('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ())
-        + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else ())
-        + (('--json',) if json else ())
+        (
+            local_path,
+            'list',
+            '::'.join((repository, list_arguments.archive))
+            if list_arguments.archive
+            else repository,
+        )
+        + (
+            ('--info',)
+            if logger.getEffectiveLevel() == logging.INFO and not list_arguments.json
+            else ()
+        )
+        + (
+            ('--debug', '--show-rc')
+            if logger.isEnabledFor(logging.DEBUG) and not list_arguments.json
+            else ()
+        )
+        + make_flags('remote-path', remote_path)
+        + make_flags('lock-wait', lock_wait)
+        + make_flags_from_arguments(list_arguments, excludes=('repository', 'archive'))
     )
 
-    return execute_command(full_command, output_log_level=None if json else logging.WARNING)
+    return execute_command(
+        full_command, output_log_level=None if list_arguments.json else logging.WARNING
+    )

borgmatic/borg/prune.py

@@ -21,12 +21,15 @@ def _make_prune_flags(retention_config):
             ('--keep-monthly', '6'),
         )
     '''
-    if not retention_config.get('prefix'):
-        retention_config['prefix'] = '{hostname}-'
+    config = retention_config.copy()
+
+    if 'prefix' not in config:
+        config['prefix'] = '{hostname}-'
+    elif not config['prefix']:
+        config.pop('prefix')
 
     return (
-        ('--' + option_name.replace('_', '-'), str(retention_config[option_name]))
-        for option_name, value in retention_config.items()
+        ('--' + option_name.replace('_', '-'), str(value)) for option_name, value in config.items()
     )
 
 

borgmatic/commands/arguments.py

@@ -108,7 +108,7 @@ def parse_arguments(*unparsed_arguments):
         type=int,
         choices=range(0, 3),
         default=0,
-        help='Display verbose progress to syslog (from none to lots: 0, 1, or 2)',
+        help='Display verbose progress to syslog (from none to lots: 0, 1, or 2). Ignored when console is interactive',
     )
     global_group.add_argument(
         '--version',
@@ -224,7 +224,7 @@ def parse_arguments(*unparsed_arguments):
     extract_group = extract_parser.add_argument_group('extract arguments')
     extract_group.add_argument(
         '--repository',
-        help='Path of repository to use, defaults to the configured repository if there is only one',
+        help='Path of repository to extract, defaults to the configured repository if there is only one',
     )
     extract_group.add_argument('--archive', help='Name of archive to operate on', required=True)
     extract_group.add_argument(
@@ -248,17 +248,48 @@ def parse_arguments(*unparsed_arguments):
         'list',
         aliases=SUBPARSER_ALIASES['list'],
         help='List archives',
-        description='List archives',
+        description='List archives or the contents of an archive',
         add_help=False,
     )
     list_group = list_parser.add_argument_group('list arguments')
     list_group.add_argument(
         '--repository',
-        help='Path of repository to use, defaults to the configured repository if there is only one',
+        help='Path of repository to list, defaults to the configured repository if there is only one',
     )
-    list_group.add_argument('--archive', help='Name of archive to operate on')
+    list_group.add_argument('--archive', help='Name of archive to list')
     list_group.add_argument(
-        '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
+        '--short', default=False, action='store_true', help='Output only archive or path names'
+    )
+    list_group.add_argument('--format', help='Format for file listing')
+    list_group.add_argument(
+        '--json', default=False, action='store_true', help='Output results as JSON'
+    )
+    list_group.add_argument(
+        '-P', '--prefix', help='Only list archive names starting with this prefix'
+    )
+    list_group.add_argument(
+        '-a', '--glob-archives', metavar='GLOB', help='Only list archive names matching this glob'
+    )
+    list_group.add_argument(
+        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
+    )
+    list_group.add_argument(
+        '--first', metavar='N', help='List first N archives after other filters are applied'
+    )
+    list_group.add_argument(
+        '--last', metavar='N', help='List first N archives after other filters are applied'
+    )
+    list_group.add_argument(
+        '-e', '--exclude', metavar='PATTERN', help='Exclude paths matching the pattern'
+    )
+    list_group.add_argument(
+        '--exclude-from', metavar='FILENAME', help='Exclude paths from exclude file, one per line'
+    )
+    list_group.add_argument('--pattern', help='Include or exclude paths matching a pattern')
+    list_group.add_argument(
+        '--pattern-from',
+        metavar='FILENAME',
+        help='Include or exclude paths matching patterns from pattern file, one per line',
     )
     list_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
@@ -270,9 +301,34 @@ def parse_arguments(*unparsed_arguments):
         add_help=False,
     )
     info_group = info_parser.add_argument_group('info arguments')
+    info_group.add_argument(
+        '--repository',
+        help='Path of repository to show info for, defaults to the configured repository if there is only one',
+    )
+    info_group.add_argument('--archive', help='Name of archive to show info for')
     info_group.add_argument(
         '--json', dest='json', default=False, action='store_true', help='Output results as JSON'
     )
+    info_group.add_argument(
+        '-P', '--prefix', help='Only show info for archive names starting with this prefix'
+    )
+    info_group.add_argument(
+        '-a',
+        '--glob-archives',
+        metavar='GLOB',
+        help='Only show info for archive names matching this glob',
+    )
+    info_group.add_argument(
+        '--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
+    )
+    info_group.add_argument(
+        '--first',
+        metavar='N',
+        help='Show info for first N archives after other filters are applied',
+    )
+    info_group.add_argument(
+        '--last', metavar='N', help='Show info for first N archives after other filters are applied'
+    )
     info_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
 
     arguments = parse_subparser_arguments(unparsed_arguments, top_level_parser, subparsers)

borgmatic/commands/borgmatic.py

@@ -171,24 +171,24 @@ def run_actions(
             json_output = borg_list.list_archives(
                 repository,
                 storage,
-                arguments['list'].archive,
+                list_arguments=arguments['list'],
                 local_path=local_path,
                 remote_path=remote_path,
-                json=arguments['list'].json,
             )
             if json_output:
                 yield json.loads(json_output)
     if 'info' in arguments:
-        logger.info('{}: Displaying summary info for archives'.format(repository))
-        json_output = borg_info.display_archives_info(
-            repository,
-            storage,
-            local_path=local_path,
-            remote_path=remote_path,
-            json=arguments['info'].json,
-        )
-        if json_output:
-            yield json.loads(json_output)
+        if arguments['info'].repository is None or repository == arguments['info'].repository:
+            logger.info('{}: Displaying summary info for archives'.format(repository))
+            json_output = borg_info.display_archives_info(
+                repository,
+                storage,
+                info_arguments=arguments['info'],
+                local_path=local_path,
+                remote_path=remote_path,
+            )
+            if json_output:
+                yield json.loads(json_output)
 
 
 def load_configurations(config_filenames):
@@ -265,7 +265,21 @@ def collect_configuration_run_summary_logs(configs, arguments):
                     msg='{}: Successfully ran configuration file'.format(config_filename),
                 )
             )
-        except (ValueError, OSError, CalledProcessError) as error:
+        except CalledProcessError as error:
+            yield logging.makeLogRecord(
+                dict(
+                    levelno=logging.CRITICAL,
+                    levelname='CRITICAL',
+                    msg='{}: Error running configuration file'.format(config_filename),
+                )
+            )
+            yield logging.makeLogRecord(
+                dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error.output)
+            )
+            yield logging.makeLogRecord(
+                dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
+            )
+        except (ValueError, OSError) as error:
             yield logging.makeLogRecord(
                 dict(
                     levelno=logging.CRITICAL,

borgmatic/config/schema.yaml

@@ -11,7 +11,7 @@ map:
             source_directories:
                 required: true
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: |
                     List of source directories to backup (required). Globs and tildes are expanded.
                 example:
@@ -21,7 +21,7 @@ map:
             repositories:
                 required: true
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: |
                     Paths to local or remote repositories (required). Tildes are expanded. Multiple
                     repositories are backed up to in sequence. See ssh_command for SSH options like
@@ -36,6 +36,18 @@ map:
                 type: bool
                 desc: Only store/extract numeric user and group identifiers. Defaults to false.
                 example: true
+            atime:
+                type: bool
+                desc: Store atime into archive. Defaults to true.
+                example: false
+            ctime:
+                type: bool
+                desc: Store ctime into archive. Defaults to true.
+                example: false
+            birthtime:
+                type: bool
+                desc: Store birthtime (creation date) into archive. Defaults to true.
+                example: false
             read_special:
                 type: bool
                 desc: |
@@ -48,23 +60,23 @@ map:
                 desc: Record bsdflags (e.g. NODUMP, IMMUTABLE) in archive. Defaults to true.
                 example: true
             files_cache:
-                type: scalar
+                type: str
                 desc: |
                     Mode in which to operate the files cache. See
                     https://borgbackup.readthedocs.io/en/stable/usage/create.html#description for
                     details. Defaults to "ctime,size,inode".
                 example: ctime,size,inode
             local_path:
-                type: scalar
+                type: str
                 desc: Alternate Borg local executable. Defaults to "borg".
                 example: borg1
             remote_path:
-                type: scalar
+                type: str
                 desc: Alternate Borg remote executable. Defaults to "borg".
                 example: borg1
             patterns:
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: |
                     Any paths matching these patterns are included/excluded from backups. Globs are
                     expanded. (Tildes are not.) Note that Borg considers this option experimental.
@@ -77,7 +89,7 @@ map:
                     - '- /home/*'
             patterns_from:
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: |
                     Read include/exclude patterns from one or more separate named files, one pattern
                     per line. Note that Borg considers this option experimental. See the output of
@@ -86,7 +98,7 @@ map:
                     - /etc/borgmatic/patterns
             exclude_patterns:
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: |
                     Any paths matching these patterns are excluded from backups. Globs and tildes
                     are expanded. See the output of "borg help patterns" for more details.
@@ -96,7 +108,7 @@ map:
                     - /etc/ssl
             exclude_from:
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: |
                     Read exclude patterns from one or more separate named files, one pattern per
                     line. See the output of "borg help patterns" for more details.
@@ -109,7 +121,7 @@ map:
                     http://www.brynosaurus.com/cachedir/spec.html for details. Defaults to false.
                 example: true
             exclude_if_present:
-                type: scalar
+                type: str
                 desc: |
                     Exclude directories that contain a file with the given filename. Defaults to not
                     set.
@@ -122,7 +134,7 @@ map:
             details.
         map:
             encryption_passcommand:
-                type: scalar
+                type: str
                 desc: |
                     The standard output of this command is used to unlock the encryption key. Only
                     use on repositories that were initialized with passcommand/repokey encryption.
@@ -130,7 +142,7 @@ map:
                     then encryption_passphrase takes precedence. Defaults to not set.
                 example: "secret-tool lookup borg-repository repo-name"
             encryption_passphrase:
-                type: scalar
+                type: str
                 desc: |
                     Passphrase to unlock the encryption key with. Only use on repositories that were
                     initialized with passphrase/repokey encryption. Quote the value if it contains
@@ -145,14 +157,14 @@ map:
                     for details. Defaults to checkpoints every 1800 seconds (30 minutes).
                 example: 1800
             chunker_params:
-                type: scalar
+                type: str
                 desc: |
                     Specify the parameters passed to then chunker (CHUNK_MIN_EXP, CHUNK_MAX_EXP,
                     HASH_MASK_BITS, HASH_WINDOW_SIZE). See https://borgbackup.readthedocs.io/en/stable/internals.html
                     for details. Defaults to "19,23,21,4095".
                 example: 19,23,21,4095
             compression:
-                type: scalar
+                type: str
                 desc: |
                     Type of compression to use when creating archives. See
                     https://borgbackup.readthedocs.org/en/stable/usage.html#borg-create for details.
@@ -163,34 +175,34 @@ map:
                 desc: Remote network upload rate limit in kiBytes/second. Defaults to unlimited.
                 example: 100
             ssh_command:
-                type: scalar
+                type: str
                 desc: |
                     Command to use instead of "ssh". This can be used to specify ssh options.
                     Defaults to not set.
                 example: ssh -i /path/to/private/key
             borg_base_directory:
-                type: scalar
+                type: str
                 desc: |
                     Base path used for various Borg directories. Defaults to $HOME, ~$USER, or ~.
                     See https://borgbackup.readthedocs.io/en/stable/usage/general.html#environment-variables for details.
                 example: /path/to/base
             borg_config_directory:
-                type: scalar
+                type: str
                 desc: |
                     Path for Borg configuration files. Defaults to $borg_base_directory/.config/borg
                 example: /path/to/base/config
             borg_cache_directory:
-                type: scalar
+                type: str
                 desc: |
                     Path for Borg cache files. Defaults to $borg_base_directory/.cache/borg
                 example: /path/to/base/cache
             borg_security_directory:
-                type: scalar
+                type: str
                 desc: |
                     Path for Borg security and encryption nonce files. Defaults to $borg_base_directory/.config/borg/security
                 example: /path/to/base/config/security
             borg_keys_directory:
-                type: scalar
+                type: str
                 desc: |
                     Path for Borg encryption key files. Defaults to $borg_base_directory/.config/borg/keys
                 example: /path/to/base/config/keys
@@ -203,7 +215,7 @@ map:
                 desc: Maximum seconds to wait for acquiring a repository/cache lock. Defaults to 1.
                 example: 5
             archive_name_format:
-                type: scalar
+                type: str
                 desc: |
                     Name of the archive. Borg placeholders can be used. See the output of
                     "borg help placeholders" for details. Defaults to
@@ -221,7 +233,7 @@ map:
             if you'd like to skip pruning entirely.
         map:
             keep_within:
-                type: scalar
+                type: str
                 desc: Keep all archives within this time interval.
                 example: 3H
             keep_secondly:
@@ -253,11 +265,11 @@ map:
                 desc: Number of yearly archives to keep.
                 example: 1
             prefix:
-                type: scalar
+                type: str
                 desc: |
                     When pruning, only consider archive names starting with this prefix.
                     Borg placeholders can be used. See the output of "borg help placeholders" for
-                    details. Defaults to "{hostname}-".
+                    details. Defaults to "{hostname}-". Use an empty value to disable the default.
                 example: sourcehostname
     consistency:
         desc: |
@@ -281,7 +293,7 @@ map:
                     - archives
             check_repositories:
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: |
                     Paths to a subset of the repositories in the location section on which to run
                     consistency checks. Handy in case some of your repositories are very large, and
@@ -295,11 +307,12 @@ map:
                       "archives" check. Defaults to checking all archives.
                 example: 3
             prefix:
-                type: scalar
+                type: str
                 desc: |
                     When performing the "archives" check, only consider archive names starting with
                     this prefix. Borg placeholders can be used. See the output of
-                    "borg help placeholders" for details. Defaults to "{hostname}-".
+                    "borg help placeholders" for details. Defaults to "{hostname}-". Use an empty
+                    value to disable the default.
                 example: sourcehostname
     output:
         desc: |
@@ -320,22 +333,24 @@ map:
         map:
             before_backup:
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: List of one or more shell commands or scripts to execute before creating a backup.
                 example:
                     - echo "Starting a backup job."
             after_backup:
                 seq:
-                    - type: scalar
+                    - type: str
                 desc: List of one or more shell commands or scripts to execute after creating a backup.
                 example:
                     - echo "Backup created."
             on_error:
                 seq:
-                    - type: scalar
-                desc: List of one or more shell commands or scripts to execute in case an exception has occurred.
+                    - type: str
+                desc: |
+                    List of one or more shell commands or scripts to execute when an exception occurs
+                    during a backup or when running a hook.
                 example:
-                    - echo "Error while creating a backup."
+                    - echo "Error while creating a backup or running a hook."
             umask:
                 type: scalar
                 desc: Umask used when executing hooks. Defaults to the umask that borgmatic is run with.

borgmatic/execute.py

@@ -4,7 +4,12 @@ import subprocess
 logger = logging.getLogger(__name__)
 
 
+ERROR_OUTPUT_MAX_LINE_COUNT = 25
+BORG_ERROR_EXIT_CODE = 2
+
+
 def execute_and_log_output(full_command, output_log_level, shell):
+    last_lines = []
     process = subprocess.Popen(
         full_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=shell
     )
@@ -14,18 +19,28 @@ def execute_and_log_output(full_command, output_log_level, shell):
         if not line:
             continue
 
-        if line.startswith('borg: error:'):
-            logger.error(line)
-        else:
-            logger.log(output_log_level, line)
+        # Keep the last few lines of output in case the command errors, and we need the output for
+        # the exception below.
+        last_lines.append(line)
+        if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT:
+            last_lines.pop(0)
+
+        logger.log(output_log_level, line)
 
     remaining_output = process.stdout.read().rstrip().decode()
     if remaining_output:  # pragma: no cover
         logger.log(output_log_level, remaining_output)
 
     exit_code = process.poll()
-    if exit_code != 0:
-        raise subprocess.CalledProcessError(exit_code, full_command)
+    if exit_code >= BORG_ERROR_EXIT_CODE:
+        # If an error occurs, include its output in the raised exception so that we don't
+        # inadvertently hide error output.
+        if len(last_lines) == ERROR_OUTPUT_MAX_LINE_COUNT:
+            last_lines.insert(0, '...')
+
+        raise subprocess.CalledProcessError(
+            exit_code, ' '.join(full_command), '\n'.join(last_lines)
+        )
 
 
 def execute_command(full_command, output_log_level=logging.INFO, shell=False):

borgmatic/logger.py

@@ -21,6 +21,14 @@ def to_bool(arg):
     return False
 
 
+def interactive_console():
+    '''
+    Return whether the current console is "interactive". Meaning: Capable of
+    user input and not just something like a cron job.
+    '''
+    return sys.stdout.isatty() and os.environ.get('TERM') != 'dumb'
+
+
 def should_do_markup(no_color, configs):
     '''
     Given the value of the command-line no-color argument, and a dict of configuration filename to
@@ -37,7 +45,7 @@ def should_do_markup(no_color, configs):
     if py_colors is not None:
         return to_bool(py_colors)
 
-    return sys.stdout.isatty() and os.environ.get('TERM') != 'dumb'
+    return interactive_console()
 
 
 LOG_LEVEL_TO_COLOR = {
@@ -82,9 +90,9 @@ def configure_logging(console_log_level, syslog_log_level=None):
     elif os.path.exists('/var/run/syslog'):
         syslog_path = '/var/run/syslog'
 
-    if syslog_path:
+    if syslog_path and not interactive_console():
         syslog_handler = logging.handlers.SysLogHandler(address=syslog_path)
-        syslog_handler.setFormatter(logging.Formatter('borgmatic: %(levelname)s \ufeff%(message)s'))
+        syslog_handler.setFormatter(logging.Formatter('borgmatic: %(levelname)s %(message)s'))
         syslog_handler.setLevel(syslog_log_level)
         handlers = (console_handler, syslog_handler)
     else:

docs/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.7.3-alpine3.9 as borgmatic
+FROM python:3.7.3-alpine3.10 as borgmatic
 
 COPY . /app
 RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
@@ -7,7 +7,7 @@ RUN borgmatic --help > /command-line.txt \
            echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \
            && borgmatic "$action" --help >> /command-line.txt; done
 
-FROM node:11.15.0-alpine as html
+FROM node:12.4.0-alpine as html
 
 WORKDIR /source
 

docs/how-to/add-preparation-and-cleanup-steps-to-backups.md

@@ -0,0 +1,58 @@
+---
+title: Add preparation and cleanup steps to backups
+---
+## Preparation and cleanup hooks
+
+If you find yourself performing prepraration tasks before your backup runs, or
+cleanup work afterwards, borgmatic hooks may be of interest. Hooks are
+shell commands that borgmatic executes for you at various points, and they're
+configured in the `hooks` section of your configuration file.
+
+For instance, you can specify `before_backup` hooks to dump a database to file
+before backing it up, and specify `after_backup` hooks to delete the temporary
+file afterwards. Here's an example:
+
+```yaml
+hooks:
+    before_backup:
+        - dump-a-database /to/file.sql
+    after_backup:
+        - rm /to/file.sql
+```
+
+borgmatic hooks run once per configuration file. `before_backup` hooks run
+prior to backups of all repositories. `after_backup` hooks run afterwards, but
+not if an error occurs in a previous hook or in the backups themselves.
+
+
+## Error hooks
+
+borgmatic also runs `on_error` hooks if an error occurs, either when creating
+a backup or running another hook. Here's an example configuration:
+
+```yaml
+hooks:
+    on_error:
+        - echo "Error while creating a backup or running a hook."
+```
+
+## Hook output
+
+Any output produced by your hooks shows up both at the console and in syslog
+(when run in a non-interactive console). For more information, read about <a
+href="https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md">inspecting
+your backups</a>.
+
+## Security
+
+An important security note about hooks: borgmatic executes all hook commands
+with the user permissions of borgmatic itself. So to prevent potential shell
+injection or privilege escalation, do not forget to set secure permissions
+(`chmod 0700`) on borgmatic configuration files and scripts invoked by hooks.
+
+
+## Related documentation
+
+ * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)
+ * [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups.md)
+ * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md)

docs/how-to/deal-with-very-large-backups.md

@@ -22,6 +22,9 @@ borgmatic create
 borgmatic check
 ```
 
+(No borgmatic `prune`, `create`, or `check` actions? Try the old-style
+`--prune`, `--create`, or `--check`. Or upgrade borgmatic!)
+
 You can run with only one of these flags provided, or you can mix and match
 any number of them in a single borgmatic run. This supports approaches like
 making backups with `create` on a frequent schedule, while only running

docs/how-to/inspect-your-backups.md

@@ -33,12 +33,16 @@ borgmatic list
 borgmatic info
 ```
 
+(No borgmatic `list` or `info` actions? Try the old-style `--list` or
+`--info`. Or upgrade borgmatic!)
+
 ## Logging
 
 By default, borgmatic logs to a local syslog-compatible daemon if one is
-present. Where those logs show up depends on your particular system. If you're
-using systemd, try running `journalctl -xe`. Otherwise, try viewing
-`/var/log/syslog` or similiar.
+present and borgmatic is running in a non-interactive console. Where those
+logs show up depends on your particular system. If you're using systemd, try
+running `journalctl -xe`. Otherwise, try viewing `/var/log/syslog` or
+similiar.
 
 You can customize the log level used for syslog logging with the
 `--syslog-verbosity` flag, and this is independent from the console logging

docs/how-to/restore-a-backup.md

@@ -11,6 +11,9 @@ to figure out which archive to restore. A good way to do that is to use the
 borgmatic list
 ```
 
+(No borgmatic `list` action? Try the old-style `--list`, or upgrade
+borgmatic!)
+
 That should yield output looking something like:
 
 ```text
@@ -25,6 +28,9 @@ and therefore the latest timestamp, run a command like:
 borgmatic extract --archive host-2019-01-02T04:06:07.080910
 ```
 
+(No borgmatic `extract` action? Try the old-style `--extract`, or upgrade
+borgmatic!)
+
 The `--archive` value is the name of the archive to restore. This extracts the
 entire contents of the archive to the current directory, so make sure you're
 in the right place before running the command.

docs/how-to/run-preparation-steps-before-backups.md

@@ -1,58 +1,3 @@
----
-title: How to run preparation steps before backups
----
-## Preparation and cleanup hooks
-
-If you find yourself performing prepraration tasks before your backup runs, or
-cleanup work afterwards, borgmatic hooks may be of interest. Hooks are
-shell commands that borgmatic executes for you at various points, and they're
-configured in the `hooks` section of your configuration file.
-
-For instance, you can specify `before_backup` hooks to dump a database to file
-before backing it up, and specify `after_backup` hooks to delete the temporary
-file afterwards. Here's an example:
-
-```yaml
-hooks:
-    before_backup:
-        - dump-a-database /to/file.sql
-    after_backup:
-        - rm /to/file.sql
-```
-
-borgmatic hooks run once per configuration file. `before_backup` hooks run
-prior to backups of all repositories. `after_backup` hooks run afterwards, but
-not if an error occurs in a previous hook or in the backups themselves.
-
-
-## Error hooks
-
-borgmatic also runs `on_error` hooks if an error occurs. Here's an example
-configuration:
-
-```yaml
-hooks:
-    on_error:
-        - echo "Error while creating a backup."
-```
-
-## Hook output
-
-Any output produced by your hooks shows up both at the console and in syslog.
-For more information, read about <a
-href="https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md">inspecting
-your backups</a>.
-
-## Security
-
-An important security note about hooks: borgmatic executes all hook commands
-with the user permissions of borgmatic itself. So to prevent potential shell
-injection or privilege escalation, do not forget to set secure permissions
-(`chmod 0700`) on borgmatic configuration files and scripts invoked by hooks.
-
-
-## Related documentation
-
- * [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups.md)
- * [Make per-application backups](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups.md)
- * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups.md)
+<head>
+    <meta http-equiv='refresh' content='0; URL=https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/'>
+</head>

docs/how-to/set-up-backups.md

@@ -38,6 +38,7 @@ borgmatic:
  * [Arch Linux](https://aur.archlinux.org/packages/borgmatic/)
  * [OpenBSD](http://ports.su/sysutils/borgmatic)
  * [openSUSE](https://software.opensuse.org/package/borgmatic)
+ * [stand-alone binary](https://github.com/cmarquardt/borgmatic-binary)
 
 
 ## Hosting providers
@@ -71,7 +72,8 @@ to ignore anything you don't need.
 Note that the configuration file is organized into distinct sections, each
 with a section name like `location:` or `storage:`. So take care that if you
 uncomment a particular option, also uncomment its containing section name, or
-else borgmatic won't recognize the option.
+else borgmatic won't recognize the option. Also be sure to use spaces rather
+than tabs for indentation; YAML does not allow tabs.
 
 You can also get the same sample configuration file from the [configuration
 reference](https://torsion.org/borgmatic/docs/reference/configuration.md), the authoritative set of
@@ -124,6 +126,9 @@ a command like the following:
 borgmatic init --encryption repokey
 ```
 
+(No borgmatic `init` action? Try the old-style `--init` flag, or upgrade
+borgmatic!)
+
 This uses the borgmatic configuration file you created above to determine
 which local or remote repository to create, and encrypts it with the
 encryption passphrase specified there if one is provided. Read about [Borg
@@ -209,6 +214,22 @@ section of configuration.
 
 ## Troubleshooting
 
+### "found character that cannot start any token" error
+
+If you run borgmatic and see an error looking something like this, it probably
+means you've used tabs instead of spaces:
+
+```
+test.yaml: Error parsing configuration file
+An error occurred while parsing a configuration file at config.yaml:
+while scanning for the next token
+found character that cannot start any token
+  in "config.yaml", line 230, column 1
+```
+
+YAML does not allow tabs. So to fix this, simply replace any tabs in your
+configuration file with the requisite number of spaces.
+
 ### libyaml compilation errors
 
 borgmatic depends on a Python YAML library (ruamel.yaml) that will optionally

scripts/find-unsupported-borg-options

@@ -48,6 +48,17 @@ for sub_command in prune create check list info; do
             | grep -v '^--stats$' \
             | grep -v '^--verbose$' \
             | grep -v '^--warning$' \
+            | grep -v '^--exclude' \
+            | grep -v '^--exclude-from' \
+            | grep -v '^--first' \
+            | grep -v '^--format' \
+            | grep -v '^--glob-archives' \
+            | grep -v '^--last' \
+            | grep -v '^--list-format' \
+            | grep -v '^--patterns-from' \
+            | grep -v '^--prefix' \
+            | grep -v '^--short' \
+            | grep -v '^--sort-by' \
             | grep -v '^-h$' \
             >> all_borg_flags
     done

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.3.9'
+VERSION = '1.3.14'
 
 
 setup(
@@ -31,7 +31,7 @@ setup(
     obsoletes=['atticmatic'],
     install_requires=(
         'pykwalify>=1.6.0,<14.06',
-        'ruamel.yaml>0.15.0,<0.16.0',
+        'ruamel.yaml>0.15.0,<0.17.0',
         'setuptools',
         'colorama>=0.4.1,<0.5',
     ),

test_requirements.txt

@@ -20,5 +20,5 @@ pytest==4.6.3
 pytest-cov==2.7.1
 python-dateutil==2.8.0
 PyYAML==5.1.1
-ruamel.yaml>0.15.0,<0.16.0
+ruamel.yaml>0.15.0,<0.17.0
 toml==0.10.0

tests/integration/test_execute.py

@@ -15,12 +15,32 @@ def test_execute_and_log_output_logs_each_line_separately():
     module.execute_and_log_output(['echo', 'there'], output_log_level=logging.INFO, shell=False)
 
 
-def test_execute_and_log_output_logs_borg_error_as_error():
-    flexmock(module.logger).should_receive('error').with_args('borg: error: oopsie').once()
+def test_execute_and_log_output_with_borg_warning_does_not_raise():
+    flexmock(module.logger).should_receive('log')
 
-    module.execute_and_log_output(
-        ['echo', 'borg: error: oopsie'], output_log_level=logging.INFO, shell=False
-    )
+    # Borg's exit code 1 is a warning, not an error.
+    module.execute_and_log_output(['false'], output_log_level=logging.INFO, shell=False)
+
+
+def test_execute_and_log_output_includes_borg_error_output_in_exception():
+    flexmock(module.logger).should_receive('log')
+
+    with pytest.raises(subprocess.CalledProcessError) as error:
+        module.execute_and_log_output(['grep'], output_log_level=logging.INFO, shell=False)
+
+    assert error.value.returncode == 2
+    assert error.value.output
+
+
+def test_execute_and_log_output_truncates_long_borg_error_output():
+    flexmock(module).ERROR_OUTPUT_MAX_LINE_COUNT = 0
+    flexmock(module.logger).should_receive('log')
+
+    with pytest.raises(subprocess.CalledProcessError) as error:
+        module.execute_and_log_output(['grep'], output_log_level=logging.INFO, shell=False)
+
+    assert error.value.returncode == 2
+    assert error.value.output.startswith('...')
 
 
 def test_execute_and_log_output_with_no_output_logs_nothing():
@@ -30,7 +50,7 @@ def test_execute_and_log_output_with_no_output_logs_nothing():
 
 
 def test_execute_and_log_output_with_error_exit_status_raises():
-    flexmock(module.logger).should_receive('log').never()
+    flexmock(module.logger).should_receive('log')
 
     with pytest.raises(subprocess.CalledProcessError):
-        module.execute_and_log_output(['false'], output_log_level=logging.INFO, shell=False)
+        module.execute_and_log_output(['grep'], output_log_level=logging.INFO, shell=False)

tests/unit/borg/test_check.py

@@ -52,14 +52,16 @@ def test_make_check_flags_with_extract_omits_extract_flag():
     assert flags == ()
 
 
-def test_make_check_flags_with_default_checks_returns_default_flags():
-    flags = module._make_check_flags(module.DEFAULT_CHECKS)
+def test_make_check_flags_with_default_checks_and_default_prefix_returns_default_flags():
+    flags = module._make_check_flags(module.DEFAULT_CHECKS, prefix=module.DEFAULT_PREFIX)
 
     assert flags == ('--prefix', module.DEFAULT_PREFIX)
 
 
-def test_make_check_flags_with_all_checks_returns_default_flags():
-    flags = module._make_check_flags(module.DEFAULT_CHECKS + ('extract',))
+def test_make_check_flags_with_all_checks_and_default_prefix_returns_default_flags():
+    flags = module._make_check_flags(
+        module.DEFAULT_CHECKS + ('extract',), prefix=module.DEFAULT_PREFIX
+    )
 
     assert flags == ('--prefix', module.DEFAULT_PREFIX)
 
@@ -67,7 +69,7 @@ def test_make_check_flags_with_all_checks_returns_default_flags():
 def test_make_check_flags_with_archives_check_and_last_includes_last_flag():
     flags = module._make_check_flags(('archives',), check_last=3)
 
-    assert flags == ('--archives-only', '--last', '3', '--prefix', module.DEFAULT_PREFIX)
+    assert flags == ('--archives-only', '--last', '3')
 
 
 def test_make_check_flags_with_repository_check_and_last_omits_last_flag():
@@ -79,7 +81,7 @@ def test_make_check_flags_with_repository_check_and_last_omits_last_flag():
 def test_make_check_flags_with_default_checks_and_last_includes_last_flag():
     flags = module._make_check_flags(module.DEFAULT_CHECKS, check_last=3)
 
-    assert flags == ('--last', '3', '--prefix', module.DEFAULT_PREFIX)
+    assert flags == ('--last', '3')
 
 
 def test_make_check_flags_with_archives_check_and_prefix_includes_prefix_flag():
@@ -88,6 +90,18 @@ def test_make_check_flags_with_archives_check_and_prefix_includes_prefix_flag():
     assert flags == ('--archives-only', '--prefix', 'foo-')
 
 
+def test_make_check_flags_with_archives_check_and_empty_prefix_omits_prefix_flag():
+    flags = module._make_check_flags(('archives',), prefix='')
+
+    assert flags == ('--archives-only',)
+
+
+def test_make_check_flags_with_archives_check_and_none_prefix_omits_prefix_flag():
+    flags = module._make_check_flags(('archives',), prefix=None)
+
+    assert flags == ('--archives-only',)
+
+
 def test_make_check_flags_with_repository_check_and_prefix_omits_prefix_flag():
     flags = module._make_check_flags(('repository',), prefix='foo-')
 
@@ -114,7 +128,7 @@ def test_check_archives_calls_borg_with_parameters(checks):
     consistency_config = {'check_last': check_last}
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').with_args(
-        checks, check_last, None
+        checks, check_last, module.DEFAULT_PREFIX
     ).and_return(())
     insert_execute_command_mock(('borg', 'check', 'repo'))
 
@@ -179,7 +193,7 @@ def test_check_archives_with_local_path_calls_borg_via_local_path():
     consistency_config = {'check_last': check_last}
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').with_args(
-        checks, check_last, None
+        checks, check_last, module.DEFAULT_PREFIX
     ).and_return(())
     insert_execute_command_mock(('borg1', 'check', 'repo'))
 
@@ -197,7 +211,7 @@ def test_check_archives_with_remote_path_calls_borg_with_remote_path_parameters(
     consistency_config = {'check_last': check_last}
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').with_args(
-        checks, check_last, None
+        checks, check_last, module.DEFAULT_PREFIX
     ).and_return(())
     insert_execute_command_mock(('borg', 'check', 'repo', '--remote-path', 'borg1'))
 
@@ -215,7 +229,7 @@ def test_check_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
     consistency_config = {'check_last': check_last}
     flexmock(module).should_receive('_parse_checks').and_return(checks)
     flexmock(module).should_receive('_make_check_flags').with_args(
-        checks, check_last, None
+        checks, check_last, module.DEFAULT_PREFIX
     ).and_return(())
     insert_execute_command_mock(('borg', 'check', 'repo', '--lock-wait', '5'))
 

tests/unit/borg/test_create.py

@@ -1,5 +1,6 @@
 import logging
 
+import pytest
 from flexmock import flexmock
 
 from borgmatic.borg import create as module
@@ -563,7 +564,8 @@ def test_create_archive_with_read_special_calls_borg_with_read_special_parameter
     )
 
 
-def test_create_archive_with_bsd_flags_true_calls_borg_without_nobsdflags_parameter():
+@pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags'))
+def test_create_archive_with_option_true_calls_borg_without_corresponding_parameter(option_name):
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
@@ -579,21 +581,22 @@ def test_create_archive_with_bsd_flags_true_calls_borg_without_nobsdflags_parame
         location_config={
             'source_directories': ['foo', 'bar'],
             'repositories': ['repo'],
-            'bsd_flags': True,
+            option_name: True,
             'exclude_patterns': None,
         },
         storage_config={},
     )
 
 
-def test_create_archive_with_bsd_flags_false_calls_borg_with_nobsdflags_parameter():
+@pytest.mark.parametrize('option_name', ('atime', 'ctime', 'birthtime', 'bsd_flags'))
+def test_create_archive_with_option_false_calls_borg_with_corresponding_parameter(option_name):
     flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar'))
     flexmock(module).should_receive('_expand_home_directories').and_return(())
     flexmock(module).should_receive('_write_pattern_file').and_return(None)
     flexmock(module).should_receive('_make_pattern_flags').and_return(())
     flexmock(module).should_receive('_make_exclude_flags').and_return(())
     flexmock(module).should_receive('execute_command').with_args(
-        CREATE_COMMAND + ('--nobsdflags',), output_log_level=logging.INFO
+        CREATE_COMMAND + ('--no' + option_name.replace('_', ''),), output_log_level=logging.INFO
     )
 
     module.create_archive(
@@ -602,7 +605,7 @@ def test_create_archive_with_bsd_flags_false_calls_borg_with_nobsdflags_paramete
         location_config={
             'source_directories': ['foo', 'bar'],
             'repositories': ['repo'],
-            'bsd_flags': False,
+            option_name: False,
             'exclude_patterns': None,
         },
         storage_config={},

tests/unit/borg/test_flags.py

@@ -0,0 +1,47 @@
+from flexmock import flexmock
+
+from borgmatic.borg import flags as module
+
+
+def test_make_flags_formats_string_value():
+    assert module.make_flags('foo', 'bar') == ('--foo', 'bar')
+
+
+def test_make_flags_formats_integer_value():
+    assert module.make_flags('foo', 3) == ('--foo', '3')
+
+
+def test_make_flags_formats_true_value():
+    assert module.make_flags('foo', True) == ('--foo',)
+
+
+def test_make_flags_omits_false_value():
+    assert module.make_flags('foo', False) == ()
+
+
+def test_make_flags_formats_name_with_underscore():
+    assert module.make_flags('posix_me_harder', 'okay') == ('--posix-me-harder', 'okay')
+
+
+def test_make_flags_from_arguments_flattens_and_sorts_multiple_arguments():
+    flexmock(module).should_receive('make_flags').with_args('foo', 'bar').and_return(('foo', 'bar'))
+    flexmock(module).should_receive('make_flags').with_args('baz', 'quux').and_return(
+        ('baz', 'quux')
+    )
+    arguments = flexmock(foo='bar', baz='quux')
+
+    assert module.make_flags_from_arguments(arguments) == ('baz', 'quux', 'foo', 'bar')
+
+
+def test_make_flags_from_arguments_excludes_underscored_argument_names():
+    flexmock(module).should_receive('make_flags').with_args('foo', 'bar').and_return(('foo', 'bar'))
+    arguments = flexmock(foo='bar', _baz='quux')
+
+    assert module.make_flags_from_arguments(arguments) == ('foo', 'bar')
+
+
+def test_make_flags_from_arguments_omits_excludes():
+    flexmock(module).should_receive('make_flags').with_args('foo', 'bar').and_return(('foo', 'bar'))
+    arguments = flexmock(foo='bar', baz='quux')
+
+    assert module.make_flags_from_arguments(arguments, excludes=('baz', 'other')) == ('foo', 'bar')

tests/unit/borg/test_info.py

@@ -1,5 +1,6 @@
 import logging
 
+import pytest
 from flexmock import flexmock
 
 from borgmatic.borg import info as module
@@ -14,7 +15,9 @@ def test_display_archives_info_calls_borg_with_parameters():
         INFO_COMMAND, output_log_level=logging.WARNING
     )
 
-    module.display_archives_info(repository='repo', storage_config={})
+    module.display_archives_info(
+        repository='repo', storage_config={}, info_arguments=flexmock(archive=None, json=False)
+    )
 
 
 def test_display_archives_info_with_log_info_calls_borg_with_info_parameter():
@@ -22,7 +25,9 @@ def test_display_archives_info_with_log_info_calls_borg_with_info_parameter():
         INFO_COMMAND + ('--info',), output_log_level=logging.WARNING
     )
     insert_logging_mock(logging.INFO)
-    module.display_archives_info(repository='repo', storage_config={})
+    module.display_archives_info(
+        repository='repo', storage_config={}, info_arguments=flexmock(archive=None, json=False)
+    )
 
 
 def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_output():
@@ -31,7 +36,9 @@ def test_display_archives_info_with_log_info_and_json_suppresses_most_borg_outpu
     ).and_return('[]')
 
     insert_logging_mock(logging.INFO)
-    json_output = module.display_archives_info(repository='repo', storage_config={}, json=True)
+    json_output = module.display_archives_info(
+        repository='repo', storage_config={}, info_arguments=flexmock(archive=None, json=True)
+    )
 
     assert json_output == '[]'
 
@@ -42,7 +49,9 @@ def test_display_archives_info_with_log_debug_calls_borg_with_debug_parameter():
     )
     insert_logging_mock(logging.DEBUG)
 
-    module.display_archives_info(repository='repo', storage_config={})
+    module.display_archives_info(
+        repository='repo', storage_config={}, info_arguments=flexmock(archive=None, json=False)
+    )
 
 
 def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_output():
@@ -51,7 +60,9 @@ def test_display_archives_info_with_log_debug_and_json_suppresses_most_borg_outp
     ).and_return('[]')
 
     insert_logging_mock(logging.DEBUG)
-    json_output = module.display_archives_info(repository='repo', storage_config={}, json=True)
+    json_output = module.display_archives_info(
+        repository='repo', storage_config={}, info_arguments=flexmock(archive=None, json=True)
+    )
 
     assert json_output == '[]'
 
@@ -61,17 +72,34 @@ def test_display_archives_info_with_json_calls_borg_with_json_parameter():
         INFO_COMMAND + ('--json',), output_log_level=None
     ).and_return('[]')
 
-    json_output = module.display_archives_info(repository='repo', storage_config={}, json=True)
+    json_output = module.display_archives_info(
+        repository='repo', storage_config={}, info_arguments=flexmock(archive=None, json=True)
+    )
 
     assert json_output == '[]'
 
 
+def test_display_archives_info_with_archive_calls_borg_with_archive_parameter():
+    flexmock(module).should_receive('execute_command').with_args(
+        ('borg', 'info', 'repo::archive'), output_log_level=logging.WARNING
+    )
+
+    module.display_archives_info(
+        repository='repo', storage_config={}, info_arguments=flexmock(archive='archive', json=False)
+    )
+
+
 def test_display_archives_info_with_local_path_calls_borg_via_local_path():
     flexmock(module).should_receive('execute_command').with_args(
         ('borg1',) + INFO_COMMAND[1:], output_log_level=logging.WARNING
     )
 
-    module.display_archives_info(repository='repo', storage_config={}, local_path='borg1')
+    module.display_archives_info(
+        repository='repo',
+        storage_config={},
+        info_arguments=flexmock(archive=None, json=False),
+        local_path='borg1',
+    )
 
 
 def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_parameters():
@@ -79,7 +107,12 @@ def test_display_archives_info_with_remote_path_calls_borg_with_remote_path_para
         INFO_COMMAND + ('--remote-path', 'borg1'), output_log_level=logging.WARNING
     )
 
-    module.display_archives_info(repository='repo', storage_config={}, remote_path='borg1')
+    module.display_archives_info(
+        repository='repo',
+        storage_config={},
+        info_arguments=flexmock(archive=None, json=False),
+        remote_path='borg1',
+    )
 
 
 def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_parameters():
@@ -88,4 +121,22 @@ def test_display_archives_info_with_lock_wait_calls_borg_with_lock_wait_paramete
         INFO_COMMAND + ('--lock-wait', '5'), output_log_level=logging.WARNING
     )
 
-    module.display_archives_info(repository='repo', storage_config=storage_config)
+    module.display_archives_info(
+        repository='repo',
+        storage_config=storage_config,
+        info_arguments=flexmock(archive=None, json=False),
+    )
+
+
+@pytest.mark.parametrize('argument_name', ('prefix', 'glob_archives', 'sort_by', 'first', 'last'))
+def test_display_archives_info_passes_through_arguments_to_borg(argument_name):
+    flexmock(module).should_receive('execute_command').with_args(
+        INFO_COMMAND + ('--' + argument_name.replace('_', '-'), 'value'),
+        output_log_level=logging.WARNING,
+    )
+
+    module.display_archives_info(
+        repository='repo',
+        storage_config={},
+        info_arguments=flexmock(archive=None, json=False, **{argument_name: 'value'}),
+    )

tests/unit/borg/test_init.py

@@ -35,6 +35,25 @@ def test_initialize_repository_calls_borg_with_parameters():
     module.initialize_repository(repository='repo', encryption_mode='repokey')
 
 
+def test_initialize_repository_does_not_raise_for_borg_init_warning():
+    insert_info_command_not_found_mock()
+    flexmock(module.subprocess).should_receive('check_call').and_raise(
+        module.subprocess.CalledProcessError(1, 'borg init')
+    )
+
+    module.initialize_repository(repository='repo', encryption_mode='repokey')
+
+
+def test_initialize_repository_raises_for_borg_init_error():
+    insert_info_command_not_found_mock()
+    flexmock(module.subprocess).should_receive('check_call').and_raise(
+        module.subprocess.CalledProcessError(2, 'borg init')
+    )
+
+    with pytest.raises(subprocess.CalledProcessError):
+        module.initialize_repository(repository='repo', encryption_mode='repokey')
+
+
 def test_initialize_repository_skips_initialization_when_repository_already_exists():
     insert_info_command_found_mock()
     flexmock(module.subprocess).should_receive('check_call').never()

tests/unit/borg/test_list.py

@@ -1,5 +1,6 @@
 import logging
 
+import pytest
 from flexmock import flexmock
 
 from borgmatic.borg import list as module
@@ -14,7 +15,9 @@ def test_list_archives_calls_borg_with_parameters():
         LIST_COMMAND, output_log_level=logging.WARNING
     )
 
-    module.list_archives(repository='repo', storage_config={})
+    module.list_archives(
+        repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=False)
+    )
 
 
 def test_list_archives_with_log_info_calls_borg_with_info_parameter():
@@ -23,7 +26,9 @@ def test_list_archives_with_log_info_calls_borg_with_info_parameter():
     )
     insert_logging_mock(logging.INFO)
 
-    module.list_archives(repository='repo', storage_config={})
+    module.list_archives(
+        repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=False)
+    )
 
 
 def test_list_archives_with_log_info_and_json_suppresses_most_borg_output():
@@ -32,7 +37,9 @@ def test_list_archives_with_log_info_and_json_suppresses_most_borg_output():
     )
     insert_logging_mock(logging.INFO)
 
-    module.list_archives(repository='repo', storage_config={}, json=True)
+    module.list_archives(
+        repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=True)
+    )
 
 
 def test_list_archives_with_log_debug_calls_borg_with_debug_parameter():
@@ -41,7 +48,9 @@ def test_list_archives_with_log_debug_calls_borg_with_debug_parameter():
     )
     insert_logging_mock(logging.DEBUG)
 
-    module.list_archives(repository='repo', storage_config={})
+    module.list_archives(
+        repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=False)
+    )
 
 
 def test_list_archives_with_log_debug_and_json_suppresses_most_borg_output():
@@ -50,7 +59,9 @@ def test_list_archives_with_log_debug_and_json_suppresses_most_borg_output():
     )
     insert_logging_mock(logging.DEBUG)
 
-    module.list_archives(repository='repo', storage_config={}, json=True)
+    module.list_archives(
+        repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=True)
+    )
 
 
 def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
@@ -59,7 +70,11 @@ def test_list_archives_with_lock_wait_calls_borg_with_lock_wait_parameters():
         LIST_COMMAND + ('--lock-wait', '5'), output_log_level=logging.WARNING
     )
 
-    module.list_archives(repository='repo', storage_config=storage_config)
+    module.list_archives(
+        repository='repo',
+        storage_config=storage_config,
+        list_arguments=flexmock(archive=None, json=False),
+    )
 
 
 def test_list_archives_with_archive_calls_borg_with_archive_parameter():
@@ -68,7 +83,11 @@ def test_list_archives_with_archive_calls_borg_with_archive_parameter():
         ('borg', 'list', 'repo::archive'), output_log_level=logging.WARNING
     )
 
-    module.list_archives(repository='repo', storage_config=storage_config, archive='archive')
+    module.list_archives(
+        repository='repo',
+        storage_config=storage_config,
+        list_arguments=flexmock(archive='archive', json=False),
+    )
 
 
 def test_list_archives_with_local_path_calls_borg_via_local_path():
@@ -76,7 +95,12 @@ def test_list_archives_with_local_path_calls_borg_via_local_path():
         ('borg1',) + LIST_COMMAND[1:], output_log_level=logging.WARNING
     )
 
-    module.list_archives(repository='repo', storage_config={}, local_path='borg1')
+    module.list_archives(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(archive=None, json=False),
+        local_path='borg1',
+    )
 
 
 def test_list_archives_with_remote_path_calls_borg_with_remote_path_parameters():
@@ -84,7 +108,51 @@ def test_list_archives_with_remote_path_calls_borg_with_remote_path_parameters()
         LIST_COMMAND + ('--remote-path', 'borg1'), output_log_level=logging.WARNING
     )
 
-    module.list_archives(repository='repo', storage_config={}, remote_path='borg1')
+    module.list_archives(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(archive=None, json=False),
+        remote_path='borg1',
+    )
+
+
+def test_list_archives_with_short_calls_borg_with_short_parameter():
+    flexmock(module).should_receive('execute_command').with_args(
+        LIST_COMMAND + ('--short',), output_log_level=logging.WARNING
+    ).and_return('[]')
+
+    module.list_archives(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(archive=None, json=False, short=True),
+    )
+
+
+@pytest.mark.parametrize(
+    'argument_name',
+    (
+        'prefix',
+        'glob_archives',
+        'sort_by',
+        'first',
+        'last',
+        'exclude',
+        'exclude_from',
+        'pattern',
+        'pattern_from',
+    ),
+)
+def test_list_archives_passes_through_arguments_to_borg(argument_name):
+    flexmock(module).should_receive('execute_command').with_args(
+        LIST_COMMAND + ('--' + argument_name.replace('_', '-'), 'value'),
+        output_log_level=logging.WARNING,
+    ).and_return('[]')
+
+    module.list_archives(
+        repository='repo',
+        storage_config={},
+        list_arguments=flexmock(archive=None, json=False, **{argument_name: 'value'}),
+    )
 
 
 def test_list_archives_with_json_calls_borg_with_json_parameter():
@@ -92,6 +160,8 @@ def test_list_archives_with_json_calls_borg_with_json_parameter():
         LIST_COMMAND + ('--json',), output_log_level=None
     ).and_return('[]')
 
-    json_output = module.list_archives(repository='repo', storage_config={}, json=True)
+    json_output = module.list_archives(
+        repository='repo', storage_config={}, list_arguments=flexmock(archive=None, json=True)
+    )
 
     assert json_output == '[]'

tests/unit/borg/test_prune.py

@@ -33,6 +33,26 @@ def test_make_prune_flags_accepts_prefix_with_placeholders():
     assert tuple(result) == expected
 
 
+def test_make_prune_flags_treats_empty_prefix_as_no_prefix():
+    retention_config = OrderedDict((('keep_daily', 1), ('prefix', '')))
+
+    result = module._make_prune_flags(retention_config)
+
+    expected = (('--keep-daily', '1'),)
+
+    assert tuple(result) == expected
+
+
+def test_make_prune_flags_treats_none_prefix_as_no_prefix():
+    retention_config = OrderedDict((('keep_daily', 1), ('prefix', None)))
+
+    result = module._make_prune_flags(retention_config)
+
+    expected = (('--keep-daily', '1'),)
+
+    assert tuple(result) == expected
+
+
 PRUNE_COMMAND = (
     'borg',
     'prune',

tests/unit/commands/test_borgmatic.py

@@ -1,3 +1,5 @@
+import subprocess
+
 from flexmock import flexmock
 
 from borgmatic.commands import borgmatic as module
@@ -85,7 +87,7 @@ def test_collect_configuration_run_summary_logs_info_for_success_with_list():
     assert all(log for log in logs if log.levelno == module.logging.INFO)
 
 
-def test_collect_configuration_run_summary_logs_critical_for_run_error():
+def test_collect_configuration_run_summary_logs_critical_for_run_value_error():
     flexmock(module.validate).should_receive('guard_configuration_contains_repository')
     flexmock(module).should_receive('run_configuration').and_raise(ValueError)
     arguments = {}
@@ -97,6 +99,21 @@ def test_collect_configuration_run_summary_logs_critical_for_run_error():
     assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
 
 
+def test_collect_configuration_run_summary_logs_critical_including_output_for_run_process_error():
+    flexmock(module.validate).should_receive('guard_configuration_contains_repository')
+    flexmock(module).should_receive('run_configuration').and_raise(
+        subprocess.CalledProcessError(1, 'command', 'error output')
+    )
+    arguments = {}
+
+    logs = tuple(
+        module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
+    )
+
+    assert any(log for log in logs if log.levelno == module.logging.CRITICAL)
+    assert any(log for log in logs if 'error output' in str(log))
+
+
 def test_collect_configuration_run_summary_logs_outputs_merged_json_results():
     flexmock(module).should_receive('run_configuration').and_return(['foo', 'bar']).and_return(
         ['baz']

tests/unit/test_logger.py

@@ -20,6 +20,29 @@ def test_to_bool_passes_none_through():
     assert module.to_bool(None) is None
 
 
+def test_interactive_console_false_when_not_isatty(capsys):
+    with capsys.disabled():
+        flexmock(module.sys.stdout).should_receive('isatty').and_return(False)
+
+        assert module.interactive_console() is False
+
+
+def test_interactive_console_false_when_TERM_is_dumb(capsys):
+    with capsys.disabled():
+        flexmock(module.sys.stdout).should_receive('isatty').and_return(True)
+        flexmock(module.os.environ).should_receive('get').with_args('TERM').and_return('dumb')
+
+        assert module.interactive_console() is False
+
+
+def test_interactive_console_true_when_isatty_and_TERM_is_not_dumb(capsys):
+    with capsys.disabled():
+        flexmock(module.sys.stdout).should_receive('isatty').and_return(True)
+        flexmock(module.os.environ).should_receive('get').with_args('TERM').and_return('smart')
+
+        assert module.interactive_console() is True
+
+
 def test_should_do_markup_respects_no_color_value():
     assert module.should_do_markup(no_color=True, configs={}) is False
 
@@ -75,15 +98,17 @@ def test_should_do_markup_prefers_no_color_value_to_PY_COLORS():
     assert module.should_do_markup(no_color=True, configs={}) is False
 
 
-def test_should_do_markup_respects_stdout_tty_value():
+def test_should_do_markup_respects_interactive_console_value():
     flexmock(module.os.environ).should_receive('get').and_return(None)
+    flexmock(module).should_receive('interactive_console').and_return(True)
 
-    assert module.should_do_markup(no_color=False, configs={}) is False
+    assert module.should_do_markup(no_color=False, configs={}) is True
 
 
-def test_should_do_markup_prefers_PY_COLORS_to_stdout_tty_value():
+def test_should_do_markup_prefers_PY_COLORS_to_interactive_console_value():
     flexmock(module.os.environ).should_receive('get').and_return('True')
     flexmock(module).should_receive('to_bool').and_return(True)
+    flexmock(module).should_receive('interactive_console').and_return(False)
 
     assert module.should_do_markup(no_color=False, configs={}) is True
 
@@ -108,11 +133,11 @@ def test_color_text_without_color_does_not_raise():
 
 def test_configure_logging_probes_for_log_socket_on_linux():
     flexmock(module).should_receive('Console_color_formatter')
+    flexmock(module).should_receive('interactive_console').and_return(False)
     flexmock(module.logging).should_receive('basicConfig').with_args(
         level=logging.INFO, handlers=tuple
     )
     flexmock(module.os.path).should_receive('exists').with_args('/dev/log').and_return(True)
-    flexmock(module.os.path).should_receive('exists').with_args('/var/run/syslog').and_return(False)
     syslog_handler = logging.handlers.SysLogHandler()
     flexmock(module.logging.handlers).should_receive('SysLogHandler').with_args(
         address='/dev/log'
@@ -123,6 +148,7 @@ def test_configure_logging_probes_for_log_socket_on_linux():
 
 def test_configure_logging_probes_for_log_socket_on_macos():
     flexmock(module).should_receive('Console_color_formatter')
+    flexmock(module).should_receive('interactive_console').and_return(False)
     flexmock(module.logging).should_receive('basicConfig').with_args(
         level=logging.INFO, handlers=tuple
     )
@@ -155,3 +181,15 @@ def test_configure_logging_skips_syslog_if_not_found():
     flexmock(module.logging.handlers).should_receive('SysLogHandler').never()
 
     module.configure_logging(console_log_level=logging.INFO)
+
+
+def test_configure_logging_skips_syslog_if_interactive_console():
+    flexmock(module).should_receive('Console_color_formatter')
+    flexmock(module).should_receive('interactive_console').and_return(True)
+    flexmock(module.logging).should_receive('basicConfig').with_args(
+        level=logging.INFO, handlers=tuple
+    )
+    flexmock(module.os.path).should_receive('exists').with_args('/dev/log').and_return(True)
+    flexmock(module.logging.handlers).should_receive('SysLogHandler').never()
+
+    module.configure_logging(console_log_level=logging.INFO)