Fix regression of generate-borgmatic-config working without --source flag (#239).

.drone.yml

@@ -40,6 +40,16 @@ steps:
     - scripts/run-tests
 ---
 kind: pipeline
+name: python-3-8-alpine-3-10
+
+steps:
+- name: build
+  image: python:3.8-alpine3.10
+  pull: always
+  commands:
+    - scripts/run-tests
+---
+kind: pipeline
 name: documentation
 
 steps:

NEWS

@@ -1,3 +1,22 @@
+1.4.7
+ * #238: In documentation, clarify when Healthchecks/Cronitor hooks fire in relation to other hooks.
+ * #239: Upgrade your borgmatic configuration to get new options and comments via
+   "generate-borgmatic-config --source". See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/upgrade/#upgrading-your-configuration
+
+1.4.6
+ * Verbosity level "-1" for even quieter output: Errors only (#236).
+
+1.4.5
+ * Log to file instead of syslog via command-line "--log-file" flag (#233).
+
+1.4.4
+ * #234: Support for Borg --keep-exclude-tags and --exclude-nodump options.
+
+1.4.3
+ * Monitor backups with Cronitor hook integration. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook
+
 1.4.2
  * Extract files to a particular directory via "borgmatic extract --destination" flag.
  * Rename "borgmatic extract --restore-path" flag to "--path" to reduce confusion with the separate

borgmatic/borg/create.py

@@ -90,8 +90,18 @@ def _make_exclude_flags(location_config, exclude_filename=None):
     caches_flag = ('--exclude-caches',) if location_config.get('exclude_caches') else ()
     if_present = location_config.get('exclude_if_present')
     if_present_flags = ('--exclude-if-present', if_present) if if_present else ()
+    keep_exclude_tags_flags = (
+        ('--keep-exclude-tags',) if location_config.get('keep_exclude_tags') else ()
+    )
+    exclude_nodump_flags = ('--exclude-nodump',) if location_config.get('exclude_nodump') else ()
 
-    return exclude_from_flags + caches_flag + if_present_flags
+    return (
+        exclude_from_flags
+        + caches_flag
+        + if_present_flags
+        + keep_exclude_tags_flags
+        + exclude_nodump_flags
+    )
 
 
 BORGMATIC_SOURCE_DIRECTORY = '~/.borgmatic'

borgmatic/commands/arguments.py

@@ -138,16 +138,29 @@ def parse_arguments(*unparsed_arguments):
         '-v',
         '--verbosity',
         type=int,
-        choices=range(0, 3),
+        choices=range(-1, 3),
         default=0,
-        help='Display verbose progress to the console (from none to lots: 0, 1, or 2)',
+        help='Display verbose progress to the console (from none to lots: 0, 1, or 2) or only errors (-1)',
     )
     global_group.add_argument(
         '--syslog-verbosity',
         type=int,
-        choices=range(0, 3),
+        choices=range(-1, 3),
         default=0,
-        help='Display verbose progress to syslog (from none to lots: 0, 1, or 2). Ignored when console is interactive',
+        help='Log verbose progress to syslog (from none to lots: 0, 1, or 2) or only errors (-1). Ignored when console is interactive or --log-file is given',
+    )
+    global_group.add_argument(
+        '--log-file-verbosity',
+        type=int,
+        choices=range(-1, 3),
+        default=0,
+        help='Log verbose progress to log file (from none to lots: 0, 1, or 2) or only errors (-1). Only used when --log-file is given',
+    )
+    global_group.add_argument(
+        '--log-file',
+        type=str,
+        default=None,
+        help='Write log messages to this file instead of syslog',
     )
     global_group.add_argument(
         '--version',
@@ -273,7 +286,7 @@ def parse_arguments(*unparsed_arguments):
         '--repository',
         help='Path of repository to extract, defaults to the configured repository if there is only one',
     )
-    extract_group.add_argument('--archive', help='Name of archive to operate on', required=True)
+    extract_group.add_argument('--archive', help='Name of archive to extract', required=True)
     extract_group.add_argument(
         '--path',
         '--restore-path',
@@ -311,7 +324,7 @@ def parse_arguments(*unparsed_arguments):
         '--repository',
         help='Path of repository to restore from, defaults to the configured repository if there is only one',
     )
-    restore_group.add_argument('--archive', help='Name of archive to operate on', required=True)
+    restore_group.add_argument('--archive', help='Name of archive to restore from', required=True)
     restore_group.add_argument(
         '--database',
         metavar='NAME',

borgmatic/commands/borgmatic.py

@@ -18,7 +18,7 @@ from borgmatic.borg import list as borg_list
 from borgmatic.borg import prune as borg_prune
 from borgmatic.commands.arguments import parse_arguments
 from borgmatic.config import checks, collect, convert, validate
-from borgmatic.hooks import command, healthchecks, postgresql
+from borgmatic.hooks import command, cronitor, healthchecks, postgresql
 from borgmatic.logger import configure_logging, should_do_markup
 from borgmatic.signals import configure_signals
 from borgmatic.verbosity import verbosity_to_log_level
@@ -56,6 +56,9 @@ def run_configuration(config_filename, config, arguments):
             healthchecks.ping_healthchecks(
                 hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'start'
             )
+            cronitor.ping_cronitor(
+                hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'run'
+            )
             command.execute_hook(
                 hooks.get('before_backup'),
                 hooks.get('umask'),
@@ -108,6 +111,9 @@ def run_configuration(config_filename, config, arguments):
             healthchecks.ping_healthchecks(
                 hooks.get('healthchecks'), config_filename, global_arguments.dry_run
             )
+            cronitor.ping_cronitor(
+                hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'complete'
+            )
         except (OSError, CalledProcessError) as error:
             encountered_error = error
             yield from make_error_log_records(
@@ -129,6 +135,9 @@ def run_configuration(config_filename, config, arguments):
             healthchecks.ping_healthchecks(
                 hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'fail'
             )
+            cronitor.ping_cronitor(
+                hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'fail'
+            )
         except (OSError, CalledProcessError) as error:
             yield from make_error_log_records(
                 '{}: Error running on-error hook'.format(config_filename), error
@@ -475,10 +484,17 @@ def main():  # pragma: no cover
     configs, parse_logs = load_configurations(config_filenames)
 
     colorama.init(autoreset=True, strip=not should_do_markup(global_arguments.no_color, configs))
-    configure_logging(
-        verbosity_to_log_level(global_arguments.verbosity),
-        verbosity_to_log_level(global_arguments.syslog_verbosity),
-    )
+    try:
+        configure_logging(
+            verbosity_to_log_level(global_arguments.verbosity),
+            verbosity_to_log_level(global_arguments.syslog_verbosity),
+            verbosity_to_log_level(global_arguments.log_file_verbosity),
+            global_arguments.log_file,
+        )
+    except (FileNotFoundError, PermissionError) as error:
+        configure_logging(logging.CRITICAL)
+        logger.critical('Error configuring logging: {}'.format(error))
+        exit_with_help_link()
 
     logger.debug('Ensuring legacy configuration is upgraded')
     convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH, config_filenames)

borgmatic/commands/generate_config.py

@@ -12,12 +12,18 @@ def parse_arguments(*arguments):
     them as an ArgumentParser instance.
     '''
     parser = ArgumentParser(description='Generate a sample borgmatic YAML configuration file.')
+    parser.add_argument(
+        '-s',
+        '--source',
+        dest='source_filename',
+        help='Optional YAML configuration file to merge into the generated configuration, useful for upgrading your configuration',
+    )
     parser.add_argument(
         '-d',
         '--destination',
         dest='destination_filename',
         default=DEFAULT_DESTINATION_CONFIG_FILENAME,
-        help='Destination YAML configuration filename. Default: {}'.format(
+        help='Destination YAML configuration file. Default: {}'.format(
             DEFAULT_DESTINATION_CONFIG_FILENAME
         ),
     )
@@ -30,11 +36,21 @@ def main():  # pragma: no cover
         args = parse_arguments(*sys.argv[1:])
 
         generate.generate_sample_configuration(
-            args.destination_filename, validate.schema_filename()
+            args.source_filename, args.destination_filename, validate.schema_filename()
         )
 
         print('Generated a sample configuration file at {}.'.format(args.destination_filename))
         print()
+        if args.source_filename:
+            print(
+                'Merged in the contents of configuration file at {}.'.format(args.source_filename)
+            )
+            print('To review the changes made, run:')
+            print()
+            print(
+                '    diff --unified {} {}'.format(args.source_filename, args.destination_filename)
+            )
+            print()
         print('Please edit the file to suit your needs. The values are representative.')
         print('All fields are optional except where indicated.')
         print()

borgmatic/config/generate.py

@@ -1,9 +1,12 @@
+import collections
 import io
 import os
 import re
 
 from ruamel import yaml
 
+from borgmatic.config import load
+
 INDENT = 4
 SEQUENCE_INDENT = 2
 
@@ -40,8 +43,8 @@ def _schema_to_sample_configuration(schema, level=0, parent_is_sequence=False):
     elif 'map' in schema:
         config = yaml.comments.CommentedMap(
             [
-                (section_name, _schema_to_sample_configuration(section_schema, level + 1))
-                for section_name, section_schema in schema['map'].items()
+                (field_name, _schema_to_sample_configuration(sub_schema, level + 1))
+                for field_name, sub_schema in schema['map'].items()
             ]
         )
         indent = (level * INDENT) + (SEQUENCE_INDENT if parent_is_sequence else 0)
@@ -68,37 +71,32 @@ def _comment_out_line(line):
     return '# '.join((indent_spaces, line[count_indent_spaces:]))
 
 
-REQUIRED_KEYS = {'source_directories', 'repositories', 'keep_daily'}
-REQUIRED_SECTION_NAMES = {'location', 'retention'}
-
-
 def _comment_out_optional_configuration(rendered_config):
     '''
-    Post-process a rendered configuration string to comment out optional key/values. The idea is
-    that this prevents the user from having to comment out a bunch of configuration they don't care
-    about to get to a minimal viable configuration file.
+    Post-process a rendered configuration string to comment out optional key/values, as determined
+    by a sentinel in the comment before each key.
 
-    Ideally ruamel.yaml would support this during configuration generation, but it's not terribly
-    easy to accomplish that way.
+    The idea is that the pre-commented configuration prevents the user from having to comment out a
+    bunch of configuration they don't care about to get to a minimal viable configuration file.
+
+    Ideally ruamel.yaml would support commenting out keys during configuration generation, but it's
+    not terribly easy to accomplish that way.
     '''
     lines = []
-    required = False
+    optional = False
 
     for line in rendered_config.split('\n'):
-        key = line.strip().split(':')[0]
-
-        if key in REQUIRED_SECTION_NAMES:
-            lines.append(line)
+        # Upon encountering an optional configuration option, commenting out lines until the next
+        # blank line.
+        if line.strip().startswith('# {}'.format(COMMENTED_OUT_SENTINEL)):
+            optional = True
             continue
 
-        # Upon encountering a required configuration option, skip commenting out lines until the
-        # next blank line.
-        if key in REQUIRED_KEYS:
-            required = True
-        elif not key:
-            required = False
+        # Hit a blank line, so reset commenting.
+        if not line.strip():
+            optional = False
 
-        lines.append(_comment_out_line(line) if not required else line)
+        lines.append(_comment_out_line(line) if optional else line)
 
     return '\n'.join(lines)
 
@@ -168,6 +166,11 @@ def add_comments_to_configuration_sequence(config, schema, indent=0):
         return
 
 
+REQUIRED_SECTION_NAMES = {'location', 'retention'}
+REQUIRED_KEYS = {'source_directories', 'repositories', 'keep_daily'}
+COMMENTED_OUT_SENTINEL = 'COMMENT_OUT'
+
+
 def add_comments_to_configuration_map(config, schema, indent=0, skip_first=False):
     '''
     Using descriptions from a schema as a source, add those descriptions as comments to the given
@@ -178,10 +181,20 @@ def add_comments_to_configuration_map(config, schema, indent=0, skip_first=False
             continue
 
         field_schema = schema['map'].get(field_name, {})
-        description = field_schema.get('desc')
+        description = field_schema.get('desc', '').strip()
+
+        # If this is an optional key, add an indicator to the comment flagging it to be commented
+        # out from the sample configuration. This sentinel is consumed by downstream processing that
+        # does the actual commenting out.
+        if field_name not in REQUIRED_SECTION_NAMES and field_name not in REQUIRED_KEYS:
+            description = (
+                '\n'.join((description, COMMENTED_OUT_SENTINEL))
+                if description
+                else COMMENTED_OUT_SENTINEL
+            )
 
         # No description to use? Skip it.
-        if not field_schema or not description:
+        if not field_schema or not description:  # pragma: no cover
             continue
 
         config.yaml_set_comment_before_after_key(key=field_name, before=description, indent=indent)
@@ -190,14 +203,88 @@ def add_comments_to_configuration_map(config, schema, indent=0, skip_first=False
             _insert_newline_before_comment(config, field_name)
 
 
-def generate_sample_configuration(config_filename, schema_filename):
+RUAMEL_YAML_COMMENTS_INDEX = 1
+
+
+def remove_commented_out_sentinel(config, field_name):
     '''
-    Given a target config filename and the path to a schema filename in pykwalify YAML schema
-    format, write out a sample configuration file based on that schema.
+    Given a configuration CommentedMap and a top-level field name in it, remove any "commented out"
+    sentinel found at the end of its YAML comments. This prevents the given field name from getting
+    commented out by downstream processing that consumes the sentinel.
+    '''
+    try:
+        last_comment_value = config.ca.items[field_name][RUAMEL_YAML_COMMENTS_INDEX][-1].value
+    except KeyError:
+        return
+
+    if last_comment_value == '# {}\n'.format(COMMENTED_OUT_SENTINEL):
+        config.ca.items[field_name][RUAMEL_YAML_COMMENTS_INDEX].pop()
+
+
+def merge_source_configuration_into_destination(destination_config, source_config):
+    '''
+    Deep merge the given source configuration dict into the destination configuration CommentedMap,
+    favoring values from the source when there are collisions.
+
+    The purpose of this is to upgrade configuration files from old versions of borgmatic by adding
+    new
+    configuration keys and comments.
+    '''
+    if not source_config:
+        return destination_config
+    if not destination_config or not isinstance(source_config, collections.abc.Mapping):
+        return source_config
+
+    for field_name, source_value in source_config.items():
+        # Since this key/value is from the source configuration, leave it uncommented and remove any
+        # sentinel that would cause it to get commented out.
+        remove_commented_out_sentinel(destination_config, field_name)
+
+        # This is a mapping. Recurse for this key/value.
+        if isinstance(source_value, collections.abc.Mapping):
+            destination_config[field_name] = merge_source_configuration_into_destination(
+                destination_config[field_name], source_value
+            )
+            continue
+
+        # This is a sequence. Recurse for each item in it.
+        if isinstance(source_value, collections.abc.Sequence) and not isinstance(source_value, str):
+            destination_value = destination_config[field_name]
+            destination_config[field_name] = yaml.comments.CommentedSeq(
+                [
+                    merge_source_configuration_into_destination(
+                        destination_value[index] if index < len(destination_value) else None,
+                        source_item,
+                    )
+                    for index, source_item in enumerate(source_value)
+                ]
+            )
+            continue
+
+        # This is some sort of scalar. Simply set it into the destination.
+        destination_config[field_name] = source_config[field_name]
+
+    return destination_config
+
+
+def generate_sample_configuration(source_filename, destination_filename, schema_filename):
+    '''
+    Given an optional source configuration filename, and a required destination configuration
+    filename, and the path to a schema filename in pykwalify YAML schema format, write out a
+    sample configuration file based on that schema. If a source filename is provided, merge the
+    parsed contents of that configuration into the generated configuration.
     '''
     schema = yaml.round_trip_load(open(schema_filename))
-    config = _schema_to_sample_configuration(schema)
+    source_config = None
+
+    if source_filename:
+        source_config = load.load_configuration(source_filename)
+
+    destination_config = merge_source_configuration_into_destination(
+        _schema_to_sample_configuration(schema), source_config
+    )
 
     write_configuration(
-        config_filename, _comment_out_optional_configuration(_render_configuration(config))
+        destination_filename,
+        _comment_out_optional_configuration(_render_configuration(destination_config)),
     )

borgmatic/config/schema.yaml

@@ -126,6 +126,17 @@ map:
                     Exclude directories that contain a file with the given filename. Defaults to not
                     set.
                 example: .nobackup
+            keep_exclude_tags:
+                type: bool
+                desc: |
+                    If true, the exclude_if_present filename is included in backups. Defaults to
+                    false, meaning that the exclude_if_present filename is omitted from backups.
+                example: true
+            exclude_nodump:
+                type: bool
+                desc: |
+                    Exclude files with the NODUMP flag. Defaults to false.
+                example: true
     storage:
         desc: |
             Repository storage options. See
@@ -428,8 +439,19 @@ map:
                 desc: |
                     Healthchecks ping URL or UUID to notify when a backup begins, ends, or errors.
                     Create an account at https://healthchecks.io if you'd like to use this service.
+                    See http://localhost:8080/docs/how-to/monitor-your-backups/#healthchecks-hook
+                    for details.
                 example:
                     https://hc-ping.com/your-uuid-here
+            cronitor:
+                type: str
+                desc: |
+                    Cronitor ping URL to notify when a backup begins, ends, or errors. Create an
+                    account at https://cronitor.io if you'd like to use this service. See
+                    http://localhost:8080/docs/how-to/monitor-your-backups/#cronitor-hook for
+                    details.
+                example:
+                    https://cronitor.link/d3x0c1
             before_everything:
                 seq:
                     - type: str

borgmatic/hooks/cronitor.py

@@ -0,0 +1,24 @@
+import logging
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+def ping_cronitor(ping_url, config_filename, dry_run, append):
+    '''
+    Ping the given Cronitor URL, appending the append string. Use the given configuration filename
+    in any log entries. If this is a dry run, then don't actually ping anything.
+    '''
+    if not ping_url:
+        logger.debug('{}: No Cronitor hook set'.format(config_filename))
+        return
+
+    dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
+    ping_url = '{}/{}'.format(ping_url, append)
+
+    logger.info('{}: Pinging Cronitor {}{}'.format(config_filename, append, dry_run_label))
+    logger.debug('{}: Using Cronitor ping URL {}'.format(config_filename, ping_url))
+
+    logging.getLogger('urllib3').setLevel(logging.ERROR)
+    requests.get(ping_url)

borgmatic/hooks/healthchecks.py

@@ -7,12 +7,12 @@ logger = logging.getLogger(__name__)
 
 def ping_healthchecks(ping_url_or_uuid, config_filename, dry_run, append=None):
     '''
-    Ping the given healthchecks.io URL or UUID, appending the append string if any. Use the given
+    Ping the given Healthchecks URL or UUID, appending the append string if any. Use the given
     configuration filename in any log entries. If this is a dry run, then don't actually ping
     anything.
     '''
     if not ping_url_or_uuid:
-        logger.debug('{}: No healthchecks hook set'.format(config_filename))
+        logger.debug('{}: No Healthchecks hook set'.format(config_filename))
         return
 
     ping_url = (
@@ -26,11 +26,11 @@ def ping_healthchecks(ping_url_or_uuid, config_filename, dry_run, append=None):
         ping_url = '{}/{}'.format(ping_url, append)
 
     logger.info(
-        '{}: Pinging healthchecks.io{}{}'.format(
+        '{}: Pinging Healthchecks{}{}'.format(
             config_filename, ' ' + append if append else '', dry_run_label
         )
     )
-    logger.debug('{}: Using healthchecks.io ping URL {}'.format(config_filename, ping_url))
+    logger.debug('{}: Using Healthchecks ping URL {}'.format(config_filename, ping_url))
 
     logging.getLogger('urllib3').setLevel(logging.ERROR)
     requests.get(ping_url)

borgmatic/logger.py

@@ -73,29 +73,44 @@ def color_text(color, message):
     return '{}{}{}'.format(color, message, colorama.Style.RESET_ALL)
 
 
-def configure_logging(console_log_level, syslog_log_level=None):
+def configure_logging(
+    console_log_level, syslog_log_level=None, log_file_log_level=None, log_file=None
+):
     '''
-    Configure logging to go to both the console and syslog. Use the given log levels, respectively.
+    Configure logging to go to both the console and (syslog or log file). Use the given log levels,
+    respectively.
+
+    Raise FileNotFoundError or PermissionError if the log file could not be opened for writing.
     '''
     if syslog_log_level is None:
         syslog_log_level = console_log_level
+    if log_file_log_level is None:
+        log_file_log_level = console_log_level
 
     console_handler = logging.StreamHandler()
     console_handler.setFormatter(Console_color_formatter())
     console_handler.setLevel(console_log_level)
 
     syslog_path = None
-    if os.path.exists('/dev/log'):
-        syslog_path = '/dev/log'
-    elif os.path.exists('/var/run/syslog'):
-        syslog_path = '/var/run/syslog'
+    if log_file is None:
+        if os.path.exists('/dev/log'):
+            syslog_path = '/dev/log'
+        elif os.path.exists('/var/run/syslog'):
+            syslog_path = '/var/run/syslog'
 
     if syslog_path and not interactive_console():
         syslog_handler = logging.handlers.SysLogHandler(address=syslog_path)
         syslog_handler.setFormatter(logging.Formatter('borgmatic: %(levelname)s %(message)s'))
         syslog_handler.setLevel(syslog_log_level)
         handlers = (console_handler, syslog_handler)
+    elif log_file:
+        file_handler = logging.FileHandler(log_file)
+        file_handler.setFormatter(logging.Formatter('[%(asctime)s] %(levelname)s: %(message)s'))
+        file_handler.setLevel(log_file_log_level)
+        handlers = (console_handler, file_handler)
     else:
         handlers = (console_handler,)
 
-    logging.basicConfig(level=min(console_log_level, syslog_log_level), handlers=handlers)
+    logging.basicConfig(
+        level=min(console_log_level, syslog_log_level, log_file_log_level), handlers=handlers
+    )

borgmatic/verbosity.py

@@ -1,5 +1,6 @@
 import logging
 
+VERBOSITY_ERROR = -1
 VERBOSITY_WARNING = 0
 VERBOSITY_SOME = 1
 VERBOSITY_LOTS = 2
@@ -10,6 +11,7 @@ def verbosity_to_log_level(verbosity):
     Given a borgmatic verbosity value, return the corresponding Python log level.
     '''
     return {
+        VERBOSITY_ERROR: logging.ERROR,
         VERBOSITY_WARNING: logging.WARNING,
         VERBOSITY_SOME: logging.INFO,
         VERBOSITY_LOTS: logging.DEBUG,

docs/how-to/develop-on-borgmatic.md

@@ -62,6 +62,8 @@ following:
 tox -e black
 ```
 
+Note that Black requires at minimum Python 3.6.
+
 And if you get a complaint from the
 [isort](https://github.com/timothycrosley/isort) Python import orderer, you
 can ask isort to order your imports for you:

docs/how-to/inspect-your-backups.md

@@ -70,6 +70,20 @@ Or to increase syslog logging to include debug spew:
 borgmatic --syslog-verbosity 2
 ```
 
+### Logging to file
+
+If you don't want to use syslog, and you'd rather borgmatic log to a plain
+file, use the `--log-file` flag:
+
+```bash
+borgmatic --log-file /path/to/file.log
+```
+
+Note that if you use the `--log-file` flag, you are responsible for rotating
+the log file so it doesn't grow too large. Also, there is a
+`--log-file-verbosity` flag to customize the log file's log level.
+
+
 ### systemd journal
 
 If your local syslog daemon is systemd's journal, be aware that journald by

docs/how-to/monitor-your-backups.md

@@ -26,12 +26,15 @@ alert. But note that if borgmatic doesn't actually run, this alert won't fire.
 See [error
 hooks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hooks)
 below for how to configure this.
-4. **borgmatic Healthchecks hook**: This feature integrates with the
-[Healthchecks](https://healthchecks.io/) service, and pings Healthchecks
-whenever borgmatic runs. That way, Healthchecks can alert you when something
-goes wrong or it doesn't hear from borgmatic for a configured interval. See
+4. **borgmatic monitoring hooks**: This feature integrates with monitoring
+services like [Healthchecks](https://healthchecks.io/) and
+[Cronitor](https://cronitor.io), and pings these services whenever borgmatic
+runs. That way, you'll receive an alert when something goes wrong or the
+service doesn't hear from borgmatic for a configured interval. See
 [Healthchecks
 hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook)
+and [Cronitor
+hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook)
 below for how to configure this.
 3. **Third-party monitoring software**: You can use traditional monitoring
 software to consume borgmatic JSON output and track when the last
@@ -108,13 +111,46 @@ hooks:
     healthchecks: https://hc-ping.com/addffa72-da17-40ae-be9c-ff591afb942a
 ```
 
-With this hook in place, borgmatic will ping your Healthchecks project when a
-backup begins, ends, or errors. Then you can configure Healthchecks to notify
-you by a [variety of
+With this hook in place, borgmatic pings your Healthchecks project when a
+backup begins, ends, or errors. Specifically, before the <a
+href="https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/">`before_backup`
+hooks</a> run, borgmatic lets Healthchecks know that a backup has started.
+Then, if the backup completes successfully, borgmatic notifies Healthchecks of
+the success after the `after_backup` hooks run. And if an error occurs during
+the backup, borgmatic notifies Healthchecks after the `on_error` hooks run.
+
+You can configure Healthchecks to notify you by a [variety of
 mechanisms](https://healthchecks.io/#welcome-integrations) when backups fail
 or it doesn't hear from borgmatic for a certain period of time.
 
 
+## Cronitor hook
+
+[Cronitor](https://cronitor.io/) provides "Cron monitoring and uptime healthchecks
+for websites, services and APIs", and borgmatic has built-in
+integration with it. Once you create a Cronitor account and cron job monitor on
+their site, all you need to do is configure borgmatic with the unique "Ping
+API URL" for your monitor. Here's an example:
+
+
+```yaml
+hooks:
+    cronitor: https://cronitor.link/d3x0c1
+```
+
+With this hook in place, borgmatic pings your Cronitor monitor when a backup
+begins, ends, or errors. Specifically, before the <a
+href="https://torsion.org/borgmatic/docs/how-to/add-preparation-and-cleanup-steps-to-backups/">`before_backup`
+hooks</a> run, borgmatic lets Cronitor know that a backup has started. Then,
+if the backup completes successfully, borgmatic notifies Cronitor of the
+success after the `after_backup` hooks run. And if an error occurs during the
+backup, borgmatic notifies Cronitor after the `on_error` hooks run.
+
+You can configure Cronitor to notify you by a [variety of
+mechanisms](https://cronitor.io/docs/cron-job-notifications) when backups fail
+or it doesn't hear from borgmatic for a certain period of time.
+
+
 ## Scripting borgmatic
 
 To consume the output of borgmatic in other software, you can include an

docs/how-to/set-up-backups.md

@@ -77,9 +77,12 @@ else borgmatic won't recognize the option. Also be sure to use spaces rather
 than tabs for indentation; YAML does not allow tabs.
 
 You can also get the same sample configuration file from the [configuration
-reference](https://torsion.org/borgmatic/docs/reference/configuration/), the authoritative set of
-all configuration options. This is handy if borgmatic has added new options
-since you originally created your configuration file.
+reference](https://torsion.org/borgmatic/docs/reference/configuration/), the
+authoritative set of all configuration options. This is handy if borgmatic has
+added new options
+since you originally created your configuration file. Also check out how to
+[upgrade your
+configuration](https://torsion.org/borgmatic/docs/how-to/upgrade/#upgrading-your-configuration).
 
 
 ### Encryption
@@ -248,5 +251,6 @@ it.
  * [Deal with very large backups](https://torsion.org/borgmatic/docs/how-to/deal-with-very-large-backups/)
  * [Inspect your backups](https://torsion.org/borgmatic/docs/how-to/inspect-your-backups/)
  * [Monitor your backups](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/)
+ * [Upgrade borgmatic](https://torsion.org/borgmatic/docs/how-to/upgrade/)
  * [borgmatic configuration reference](https://torsion.org/borgmatic/docs/reference/configuration/)
  * [borgmatic command-line reference](https://torsion.org/borgmatic/docs/reference/command-line/)

docs/how-to/upgrade.md

@@ -10,7 +10,46 @@ following:
 sudo pip3 install --user --upgrade borgmatic
 ```
 
-See below about special cases.
+See below about special cases with old versions of borgmatic. Additionally, if
+you installed borgmatic [without using `pip3 install
+--user`](https://torsion.org/borgmatic/docs/how-to/set-up-backups/#other-ways-to-install),
+then your upgrade process may be different.
+
+
+### Upgrading your configuration
+
+The borgmatic configuration file format is almost always backwards-compatible
+from release to release without any changes, but you may still want to update
+your configuration file when you upgrade to take advantage of new
+configuration options. This is completely optional. If you prefer, you can add
+new configuration options manually.
+
+If you do want to upgrade your configuration file to include new options, use
+the `generate-borgmatic-config` script with its optional `--source` flag that
+takes the path to your original configuration file. If provided with this
+path, `generate-borgmatic-config` merges your original configuration into the
+generated configuration file, so you get all the newest options and comments.
+
+Here's an example:
+
+```bash
+generate-borgmatic-config --source config.yaml --destination config-new.yaml
+```
+
+New options start as commented out, so you can edit the file and decide
+whether you want to use each one.
+
+There are a few caveats to this process, however. First, when generating the
+new configuration file, `generate-borgmatic-config` replaces any comments
+you've written in your original configuration file with the newest generated
+comments. Second, the script adds back any options you had originally deleted,
+although it does so with the options commented out. And finally, any YAML
+includes you've used in the source configuration get flattened out into a
+single generated file.
+
+As a safety measure, `generate-borgmatic-config` refuses to modify
+configuration files in-place. So it's up to you to review the generated file
+and, if desired, replace your original configuration file with it.
 
 
 ### Upgrading from borgmatic 1.0.x

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.4.2'
+VERSION = '1.4.7'
 
 
 setup(

test_requirements.txt

@@ -1,12 +1,12 @@
 appdirs==1.4.3
 atomicwrites==1.3.0
-attrs==19.1.0
+attrs==19.3.0
 black==19.3b0; python_version >= '3.6'
 click==7.0
 colorama==0.4.1
 coverage==4.5.4
 docopt==0.6.2
-flake8==3.7.8
+flake8==3.7.9
 flexmock==0.10.4
 isort==4.3.21
 mccabe==0.6.1
@@ -16,8 +16,8 @@ py==1.8.0
 pycodestyle==2.5.0
 pyflakes==2.1.1
 pykwalify==1.7.0
-pytest==5.1.2
-pytest-cov==2.7.1
+pytest==5.2.2
+pytest-cov==2.8.1
 python-dateutil==2.8.0
 PyYAML==5.1.2
 requests==2.22.0

tests/integration/commands/test_arguments.py

@@ -15,6 +15,7 @@ def test_parse_arguments_with_no_arguments_uses_defaults():
     assert global_arguments.excludes_filename is None
     assert global_arguments.verbosity == 0
     assert global_arguments.syslog_verbosity == 0
+    assert global_arguments.log_file_verbosity == 0
 
 
 def test_parse_arguments_with_multiple_config_paths_parses_as_list():
@@ -26,6 +27,7 @@ def test_parse_arguments_with_multiple_config_paths_parses_as_list():
     assert global_arguments.config_paths == ['myconfig', 'otherconfig']
     assert global_arguments.verbosity == 0
     assert global_arguments.syslog_verbosity == 0
+    assert global_arguments.log_file_verbosity == 0
 
 
 def test_parse_arguments_with_verbosity_overrides_default():
@@ -39,6 +41,7 @@ def test_parse_arguments_with_verbosity_overrides_default():
     assert global_arguments.excludes_filename is None
     assert global_arguments.verbosity == 1
     assert global_arguments.syslog_verbosity == 0
+    assert global_arguments.log_file_verbosity == 0
 
 
 def test_parse_arguments_with_syslog_verbosity_overrides_default():
@@ -54,6 +57,20 @@ def test_parse_arguments_with_syslog_verbosity_overrides_default():
     assert global_arguments.syslog_verbosity == 2
 
 
+def test_parse_arguments_with_log_file_verbosity_overrides_default():
+    config_paths = ['default']
+    flexmock(module.collect).should_receive('get_default_config_paths').and_return(config_paths)
+
+    arguments = module.parse_arguments('--log-file-verbosity', '-1')
+
+    global_arguments = arguments['global']
+    assert global_arguments.config_paths == config_paths
+    assert global_arguments.excludes_filename is None
+    assert global_arguments.verbosity == 0
+    assert global_arguments.syslog_verbosity == 0
+    assert global_arguments.log_file_verbosity == -1
+
+
 def test_parse_arguments_with_list_json_overrides_default():
     arguments = module.parse_arguments('list', '--json')
 

tests/integration/config/test_generate.py

@@ -47,9 +47,13 @@ def test_comment_out_line_comments_twice_indented_option():
 
 
 def test_comment_out_optional_configuration_comments_optional_config_only():
+    # The "# COMMENT_OUT" comment is a sentinel used to express that the following key is optional.
+    # It's stripped out of the final output.
     flexmock(module)._comment_out_line = lambda line: '# ' + line
     config = '''
+# COMMENT_OUT
 foo:
+    # COMMENT_OUT
     bar:
         - baz
         - quux
@@ -59,6 +63,8 @@ location:
         - one
         - two
 
+    # This comment should be kept.
+    # COMMENT_OUT
     other: thing
     '''
 
@@ -68,12 +74,13 @@ location:
 #     bar:
 #         - baz
 #         - quux
-# 
+
 location:
     repositories:
         - one
         - two
-# 
+
+    # This comment should be kept.
 #     other: thing
     '''
 
@@ -142,12 +149,67 @@ def test_add_comments_to_configuration_map_does_not_raise():
     module.add_comments_to_configuration_map(config, schema)
 
 
+def test_add_comments_to_configuration_map_with_skip_first_does_not_raise():
+    config = module.yaml.comments.CommentedMap([('foo', 33)])
+    schema = {'map': {'foo': {'desc': 'Foo'}}}
+
+    module.add_comments_to_configuration_map(config, schema, skip_first=True)
+
+
+def test_remove_commented_out_sentinel_keeps_other_comments():
+    field_name = 'foo'
+    config = module.yaml.comments.CommentedMap([(field_name, 33)])
+    config.yaml_set_comment_before_after_key(key=field_name, before='Actual comment.\nCOMMENT_OUT')
+
+    module.remove_commented_out_sentinel(config, field_name)
+
+    comments = config.ca.items[field_name][module.RUAMEL_YAML_COMMENTS_INDEX]
+    assert len(comments) == 1
+    assert comments[0].value == '# Actual comment.\n'
+
+
+def test_remove_commented_out_sentinel_without_sentinel_keeps_other_comments():
+    field_name = 'foo'
+    config = module.yaml.comments.CommentedMap([(field_name, 33)])
+    config.yaml_set_comment_before_after_key(key=field_name, before='Actual comment.')
+
+    module.remove_commented_out_sentinel(config, field_name)
+
+    comments = config.ca.items[field_name][module.RUAMEL_YAML_COMMENTS_INDEX]
+    assert len(comments) == 1
+    assert comments[0].value == '# Actual comment.\n'
+
+
+def test_remove_commented_out_sentinel_on_unknown_field_does_not_raise():
+    field_name = 'foo'
+    config = module.yaml.comments.CommentedMap([(field_name, 33)])
+    config.yaml_set_comment_before_after_key(key=field_name, before='Actual comment.')
+
+    module.remove_commented_out_sentinel(config, 'unknown')
+
+
 def test_generate_sample_configuration_does_not_raise():
     builtins = flexmock(sys.modules['builtins'])
     builtins.should_receive('open').with_args('schema.yaml').and_return('')
+    flexmock(module.yaml).should_receive('round_trip_load')
     flexmock(module).should_receive('_schema_to_sample_configuration')
+    flexmock(module).should_receive('merge_source_configuration_into_destination')
     flexmock(module).should_receive('_render_configuration')
     flexmock(module).should_receive('_comment_out_optional_configuration')
     flexmock(module).should_receive('write_configuration')
 
-    module.generate_sample_configuration('config.yaml', 'schema.yaml')
+    module.generate_sample_configuration(None, 'dest.yaml', 'schema.yaml')
+
+
+def test_generate_sample_configuration_with_source_filename_does_not_raise():
+    builtins = flexmock(sys.modules['builtins'])
+    builtins.should_receive('open').with_args('schema.yaml').and_return('')
+    flexmock(module.yaml).should_receive('round_trip_load')
+    flexmock(module.load).should_receive('load_configuration')
+    flexmock(module).should_receive('_schema_to_sample_configuration')
+    flexmock(module).should_receive('merge_source_configuration_into_destination')
+    flexmock(module).should_receive('_render_configuration')
+    flexmock(module).should_receive('_comment_out_optional_configuration')
+    flexmock(module).should_receive('write_configuration')
+
+    module.generate_sample_configuration('source.yaml', 'dest.yaml', 'schema.yaml')

tests/unit/borg/test_create.py

@@ -150,6 +150,30 @@ def test_make_exclude_flags_includes_exclude_if_present_when_in_config():
     assert exclude_flags == ('--exclude-if-present', 'exclude_me')
 
 
+def test_make_exclude_flags_includes_keep_exclude_tags_when_true_in_config():
+    exclude_flags = module._make_exclude_flags(location_config={'keep_exclude_tags': True})
+
+    assert exclude_flags == ('--keep-exclude-tags',)
+
+
+def test_make_exclude_flags_does_not_include_keep_exclude_tags_when_false_in_config():
+    exclude_flags = module._make_exclude_flags(location_config={'keep_exclude_tags': False})
+
+    assert exclude_flags == ()
+
+
+def test_make_exclude_flags_includes_exclude_nodump_when_true_in_config():
+    exclude_flags = module._make_exclude_flags(location_config={'exclude_nodump': True})
+
+    assert exclude_flags == ('--exclude-nodump',)
+
+
+def test_make_exclude_flags_does_not_include_exclude_nodump_when_false_in_config():
+    exclude_flags = module._make_exclude_flags(location_config={'exclude_nodump': False})
+
+    assert exclude_flags == ()
+
+
 def test_make_exclude_flags_is_empty_when_config_has_no_excludes():
     exclude_flags = module._make_exclude_flags(location_config={})
 

tests/unit/config/test_convert.py

@@ -21,6 +21,7 @@ def test_convert_section_generates_integer_value_for_integer_type_in_schema():
 
 def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
     flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
+    flexmock(module.generate).should_receive('add_comments_to_configuration_map')
     source_config = Parsed_config(
         location=OrderedDict([('source_directories', '/home'), ('repository', 'hostname.borg')]),
         storage=OrderedDict([('encryption_passphrase', 'supersecret')]),
@@ -53,6 +54,7 @@ def test_convert_legacy_parsed_config_transforms_source_config_to_mapping():
 
 def test_convert_legacy_parsed_config_splits_space_separated_values():
     flexmock(module.yaml.comments).should_receive('CommentedMap').replace_with(OrderedDict)
+    flexmock(module.generate).should_receive('add_comments_to_configuration_map')
     source_config = Parsed_config(
         location=OrderedDict(
             [('source_directories', '/home /etc'), ('repository', 'hostname.borg')]

tests/unit/config/test_generate.py

@@ -51,6 +51,7 @@ def test_schema_to_sample_configuration_generates_config_sequence_of_strings_wit
 def test_schema_to_sample_configuration_generates_config_sequence_of_maps_with_examples():
     flexmock(module.yaml.comments).should_receive('CommentedSeq').replace_with(list)
     flexmock(module).should_receive('add_comments_to_configuration_sequence')
+    flexmock(module).should_receive('add_comments_to_configuration_map')
     schema = {
         'seq': [
             {
@@ -71,3 +72,63 @@ def test_schema_to_sample_configuration_with_unsupported_schema_raises():
 
     with pytest.raises(ValueError):
         module._schema_to_sample_configuration(schema)
+
+
+def test_merge_source_configuration_into_destination_inserts_map_fields():
+    destination_config = {'foo': 'dest1', 'bar': 'dest2'}
+    source_config = {'foo': 'source1', 'baz': 'source2'}
+    flexmock(module).should_receive('remove_commented_out_sentinel')
+    flexmock(module).should_receive('yaml.comments.CommentedSeq').replace_with(list)
+
+    module.merge_source_configuration_into_destination(destination_config, source_config)
+
+    assert destination_config == {'foo': 'source1', 'bar': 'dest2', 'baz': 'source2'}
+
+
+def test_merge_source_configuration_into_destination_inserts_nested_map_fields():
+    destination_config = {'foo': {'first': 'dest1', 'second': 'dest2'}, 'bar': 'dest3'}
+    source_config = {'foo': {'first': 'source1'}}
+    flexmock(module).should_receive('remove_commented_out_sentinel')
+    flexmock(module).should_receive('yaml.comments.CommentedSeq').replace_with(list)
+
+    module.merge_source_configuration_into_destination(destination_config, source_config)
+
+    assert destination_config == {'foo': {'first': 'source1', 'second': 'dest2'}, 'bar': 'dest3'}
+
+
+def test_merge_source_configuration_into_destination_inserts_sequence_fields():
+    destination_config = {'foo': ['dest1', 'dest2'], 'bar': ['dest3'], 'baz': ['dest4']}
+    source_config = {'foo': ['source1'], 'bar': ['source2', 'source3']}
+    flexmock(module).should_receive('remove_commented_out_sentinel')
+    flexmock(module).should_receive('yaml.comments.CommentedSeq').replace_with(list)
+
+    module.merge_source_configuration_into_destination(destination_config, source_config)
+
+    assert destination_config == {
+        'foo': ['source1'],
+        'bar': ['source2', 'source3'],
+        'baz': ['dest4'],
+    }
+
+
+def test_merge_source_configuration_into_destination_inserts_sequence_of_maps():
+    destination_config = {'foo': [{'first': 'dest1', 'second': 'dest2'}], 'bar': 'dest3'}
+    source_config = {'foo': [{'first': 'source1'}, {'other': 'source2'}]}
+    flexmock(module).should_receive('remove_commented_out_sentinel')
+    flexmock(module).should_receive('yaml.comments.CommentedSeq').replace_with(list)
+
+    module.merge_source_configuration_into_destination(destination_config, source_config)
+
+    assert destination_config == {
+        'foo': [{'first': 'source1', 'second': 'dest2'}, {'other': 'source2'}],
+        'bar': 'dest3',
+    }
+
+
+def test_merge_source_configuration_into_destination_without_source_does_nothing():
+    original_destination_config = {'foo': 'dest1', 'bar': 'dest2'}
+    destination_config = dict(original_destination_config)
+
+    module.merge_source_configuration_into_destination(destination_config, None)
+
+    assert destination_config == original_destination_config

tests/unit/hooks/test_cronitor.py

@@ -0,0 +1,17 @@
+from flexmock import flexmock
+
+from borgmatic.hooks import cronitor as module
+
+
+def test_ping_cronitor_hits_ping_url():
+    ping_url = 'https://example.com'
+    append = 'failed-so-hard'
+    flexmock(module.requests).should_receive('get').with_args('{}/{}'.format(ping_url, append))
+
+    module.ping_cronitor(ping_url, 'config.yaml', dry_run=False, append=append)
+
+
+def test_ping_cronitor_without_ping_url_does_not_raise():
+    flexmock(module.requests).should_receive('get').never()
+
+    module.ping_cronitor(ping_url=None, config_filename='config.yaml', dry_run=False, append='oops')

tests/unit/test_logger.py

@@ -193,3 +193,33 @@ def test_configure_logging_skips_syslog_if_interactive_console():
     flexmock(module.logging.handlers).should_receive('SysLogHandler').never()
 
     module.configure_logging(console_log_level=logging.INFO)
+
+
+def test_configure_logging_to_logfile_instead_of_syslog():
+    # syslog skipped in non-interactive console if --log-file argument provided
+    flexmock(module).should_receive('interactive_console').and_return(False)
+    flexmock(module.logging).should_receive('basicConfig').with_args(
+        level=logging.DEBUG, handlers=tuple
+    )
+    flexmock(module.os.path).should_receive('exists').with_args('/dev/log').and_return(True)
+    flexmock(module.logging.handlers).should_receive('SysLogHandler').never()
+    file_handler = logging.FileHandler('/tmp/logfile')
+    flexmock(module.logging).should_receive('FileHandler').with_args('/tmp/logfile').and_return(
+        file_handler
+    ).once()
+
+    module.configure_logging(
+        console_log_level=logging.INFO, log_file_log_level=logging.DEBUG, log_file='/tmp/logfile'
+    )
+
+
+def test_configure_logging_skips_logfile_if_argument_is_none():
+    # No FileHandler added if argument --log-file is None
+    flexmock(module).should_receive('interactive_console').and_return(False)
+    flexmock(module.logging).should_receive('basicConfig').with_args(
+        level=logging.INFO, handlers=tuple
+    )
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.logging).should_receive('FileHandler').never()
+
+    module.configure_logging(console_log_level=logging.INFO, log_file=None)

tests/unit/test_verbosity.py

@@ -17,6 +17,7 @@ def insert_logging_mock(log_level):
 def test_verbosity_to_log_level_maps_known_verbosity_to_log_level():
     assert module.verbosity_to_log_level(module.VERBOSITY_SOME) == logging.INFO
     assert module.verbosity_to_log_level(module.VERBOSITY_LOTS) == logging.DEBUG
+    assert module.verbosity_to_log_level(module.VERBOSITY_ERROR) == logging.ERROR
 
 
 def test_verbosity_to_log_level_maps_unknown_verbosity_to_warning_level():

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py35,py36,py37
+envlist = py35,py36,py37,py38
 skip_missing_interpreters = True
 skipsdist = True
 minversion = 3.14.0
@@ -14,12 +14,11 @@ commands_pre =
     find {toxinidir} -type f -not -path '{toxinidir}/.tox/*' -path '*/__pycache__/*' -name '*.py[c|o]' -delete
 commands =
     pytest {posargs}
-    py36,py37: black --check .
+    py36,py37,py38: black --check .
     isort --recursive --check-only --settings-path setup.cfg .
     flake8 borgmatic tests
 
 [testenv:black]
-basepython = python3.7
 commands =
     black {posargs} .