Handle log file error more consistently with other error. Add --log-file-verbosity flag. Add docs.

NEWS

@@ -1,3 +1,13 @@
+1.4.5
+ * Log to file instead of syslog via command-line "--log-file" flag.
+
+1.4.4
+ * #234: Support for Borg --keep-exclude-tags and --exclude-nodump options.
+
+1.4.3
+ * Monitor backups with Cronitor hook integration. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook
+
 1.4.2
  * Extract files to a particular directory via "borgmatic extract --destination" flag.
  * Rename "borgmatic extract --restore-path" flag to "--path" to reduce confusion with the separate

borgmatic/borg/create.py

@@ -90,8 +90,18 @@ def _make_exclude_flags(location_config, exclude_filename=None):
     caches_flag = ('--exclude-caches',) if location_config.get('exclude_caches') else ()
     if_present = location_config.get('exclude_if_present')
     if_present_flags = ('--exclude-if-present', if_present) if if_present else ()
+    keep_exclude_tags_flags = (
+        ('--keep-exclude-tags',) if location_config.get('keep_exclude_tags') else ()
+    )
+    exclude_nodump_flags = ('--exclude-nodump',) if location_config.get('exclude_nodump') else ()
 
-    return exclude_from_flags + caches_flag + if_present_flags
+    return (
+        exclude_from_flags
+        + caches_flag
+        + if_present_flags
+        + keep_exclude_tags_flags
+        + exclude_nodump_flags
+    )
 
 
 BORGMATIC_SOURCE_DIRECTORY = '~/.borgmatic'

borgmatic/commands/arguments.py

@@ -147,7 +147,20 @@ def parse_arguments(*unparsed_arguments):
         type=int,
         choices=range(0, 3),
         default=0,
-        help='Display verbose progress to syslog (from none to lots: 0, 1, or 2). Ignored when console is interactive',
+        help='Log verbose progress to syslog (from none to lots: 0, 1, or 2). Ignored when console is interactive or --log-file is given',
+    )
+    global_group.add_argument(
+        '--log-file-verbosity',
+        type=int,
+        choices=range(0, 3),
+        default=1,
+        help='Log verbose progress to log file (from none to lots: 0, 1, or 2). Only used when --log-file is given',
+    )
+    global_group.add_argument(
+        '--log-file',
+        type=str,
+        default=None,
+        help='Write log messages to this file instead of syslog',
     )
     global_group.add_argument(
         '--version',
@@ -273,7 +286,7 @@ def parse_arguments(*unparsed_arguments):
         '--repository',
         help='Path of repository to extract, defaults to the configured repository if there is only one',
     )
-    extract_group.add_argument('--archive', help='Name of archive to operate on', required=True)
+    extract_group.add_argument('--archive', help='Name of archive to extract', required=True)
     extract_group.add_argument(
         '--path',
         '--restore-path',
@@ -311,7 +324,7 @@ def parse_arguments(*unparsed_arguments):
         '--repository',
         help='Path of repository to restore from, defaults to the configured repository if there is only one',
     )
-    restore_group.add_argument('--archive', help='Name of archive to operate on', required=True)
+    restore_group.add_argument('--archive', help='Name of archive to restore from', required=True)
     restore_group.add_argument(
         '--database',
         metavar='NAME',

borgmatic/commands/borgmatic.py

@@ -18,7 +18,7 @@ from borgmatic.borg import list as borg_list
 from borgmatic.borg import prune as borg_prune
 from borgmatic.commands.arguments import parse_arguments
 from borgmatic.config import checks, collect, convert, validate
-from borgmatic.hooks import command, healthchecks, postgresql
+from borgmatic.hooks import command, cronitor, healthchecks, postgresql
 from borgmatic.logger import configure_logging, should_do_markup
 from borgmatic.signals import configure_signals
 from borgmatic.verbosity import verbosity_to_log_level
@@ -56,6 +56,9 @@ def run_configuration(config_filename, config, arguments):
             healthchecks.ping_healthchecks(
                 hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'start'
             )
+            cronitor.ping_cronitor(
+                hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'run'
+            )
             command.execute_hook(
                 hooks.get('before_backup'),
                 hooks.get('umask'),
@@ -108,6 +111,9 @@ def run_configuration(config_filename, config, arguments):
             healthchecks.ping_healthchecks(
                 hooks.get('healthchecks'), config_filename, global_arguments.dry_run
             )
+            cronitor.ping_cronitor(
+                hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'complete'
+            )
         except (OSError, CalledProcessError) as error:
             encountered_error = error
             yield from make_error_log_records(
@@ -129,6 +135,9 @@ def run_configuration(config_filename, config, arguments):
             healthchecks.ping_healthchecks(
                 hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'fail'
             )
+            cronitor.ping_cronitor(
+                hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'fail'
+            )
         except (OSError, CalledProcessError) as error:
             yield from make_error_log_records(
                 '{}: Error running on-error hook'.format(config_filename), error
@@ -475,10 +484,17 @@ def main():  # pragma: no cover
     configs, parse_logs = load_configurations(config_filenames)
 
     colorama.init(autoreset=True, strip=not should_do_markup(global_arguments.no_color, configs))
-    configure_logging(
-        verbosity_to_log_level(global_arguments.verbosity),
-        verbosity_to_log_level(global_arguments.syslog_verbosity),
-    )
+    try:
+        configure_logging(
+            verbosity_to_log_level(global_arguments.verbosity),
+            verbosity_to_log_level(global_arguments.syslog_verbosity),
+            verbosity_to_log_level(global_arguments.log_file_verbosity),
+            global_arguments.log_file,
+        )
+    except (FileNotFoundError, PermissionError) as error:
+        configure_logging(logging.CRITICAL)
+        logger.critical('Error configuring logging: {}'.format(error))
+        exit_with_help_link()
 
     logger.debug('Ensuring legacy configuration is upgraded')
     convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH, config_filenames)

borgmatic/config/schema.yaml

@@ -126,6 +126,17 @@ map:
                     Exclude directories that contain a file with the given filename. Defaults to not
                     set.
                 example: .nobackup
+            keep_exclude_tags:
+                type: bool
+                desc: |
+                    If true, the exclude_if_present filename is included in backups. Defaults to
+                    false, meaning that the exclude_if_present filename is omitted from backups.
+                example: true
+            exclude_nodump:
+                type: bool
+                desc: |
+                    Exclude files with the NODUMP flag. Defaults to false.
+                example: true
     storage:
         desc: |
             Repository storage options. See
@@ -430,6 +441,13 @@ map:
                     Create an account at https://healthchecks.io if you'd like to use this service.
                 example:
                     https://hc-ping.com/your-uuid-here
+            cronitor:
+                type: str
+                desc: |
+                    Cronitor ping URL to notify when a backup begins, ends, or errors. Create an
+                    account at https://cronitor.io if you'd like to use this service.
+                example:
+                    https://cronitor.link/d3x0c1
             before_everything:
                 seq:
                     - type: str

borgmatic/hooks/cronitor.py

@@ -0,0 +1,24 @@
+import logging
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+def ping_cronitor(ping_url, config_filename, dry_run, append):
+    '''
+    Ping the given Cronitor URL, appending the append string. Use the given configuration filename
+    in any log entries. If this is a dry run, then don't actually ping anything.
+    '''
+    if not ping_url:
+        logger.debug('{}: No Cronitor hook set'.format(config_filename))
+        return
+
+    dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
+    ping_url = '{}/{}'.format(ping_url, append)
+
+    logger.info('{}: Pinging Cronitor {}{}'.format(config_filename, append, dry_run_label))
+    logger.debug('{}: Using Cronitor ping URL {}'.format(config_filename, ping_url))
+
+    logging.getLogger('urllib3').setLevel(logging.ERROR)
+    requests.get(ping_url)

borgmatic/hooks/healthchecks.py

@@ -7,12 +7,12 @@ logger = logging.getLogger(__name__)
 
 def ping_healthchecks(ping_url_or_uuid, config_filename, dry_run, append=None):
     '''
-    Ping the given healthchecks.io URL or UUID, appending the append string if any. Use the given
+    Ping the given Healthchecks URL or UUID, appending the append string if any. Use the given
     configuration filename in any log entries. If this is a dry run, then don't actually ping
     anything.
     '''
     if not ping_url_or_uuid:
-        logger.debug('{}: No healthchecks hook set'.format(config_filename))
+        logger.debug('{}: No Healthchecks hook set'.format(config_filename))
         return
 
     ping_url = (
@@ -26,11 +26,11 @@ def ping_healthchecks(ping_url_or_uuid, config_filename, dry_run, append=None):
         ping_url = '{}/{}'.format(ping_url, append)
 
     logger.info(
-        '{}: Pinging healthchecks.io{}{}'.format(
+        '{}: Pinging Healthchecks{}{}'.format(
             config_filename, ' ' + append if append else '', dry_run_label
         )
     )
-    logger.debug('{}: Using healthchecks.io ping URL {}'.format(config_filename, ping_url))
+    logger.debug('{}: Using Healthchecks ping URL {}'.format(config_filename, ping_url))
 
     logging.getLogger('urllib3').setLevel(logging.ERROR)
     requests.get(ping_url)

borgmatic/logger.py

@@ -73,29 +73,44 @@ def color_text(color, message):
     return '{}{}{}'.format(color, message, colorama.Style.RESET_ALL)
 
 
-def configure_logging(console_log_level, syslog_log_level=None):
+def configure_logging(
+    console_log_level, syslog_log_level=None, log_file_log_level=None, log_file=None
+):
     '''
-    Configure logging to go to both the console and syslog. Use the given log levels, respectively.
+    Configure logging to go to both the console and (syslog or log file). Use the given log levels,
+    respectively.
+
+    Raise FileNotFoundError or PermissionError if the log file could not be opened for writing.
     '''
     if syslog_log_level is None:
         syslog_log_level = console_log_level
+    if log_file_log_level is None:
+        log_file_log_level = console_log_level
 
     console_handler = logging.StreamHandler()
     console_handler.setFormatter(Console_color_formatter())
     console_handler.setLevel(console_log_level)
 
     syslog_path = None
-    if os.path.exists('/dev/log'):
-        syslog_path = '/dev/log'
-    elif os.path.exists('/var/run/syslog'):
-        syslog_path = '/var/run/syslog'
+    if log_file is None:
+        if os.path.exists('/dev/log'):
+            syslog_path = '/dev/log'
+        elif os.path.exists('/var/run/syslog'):
+            syslog_path = '/var/run/syslog'
 
     if syslog_path and not interactive_console():
         syslog_handler = logging.handlers.SysLogHandler(address=syslog_path)
         syslog_handler.setFormatter(logging.Formatter('borgmatic: %(levelname)s %(message)s'))
         syslog_handler.setLevel(syslog_log_level)
         handlers = (console_handler, syslog_handler)
+    elif log_file:
+        file_handler = logging.FileHandler(log_file)
+        file_handler.setFormatter(logging.Formatter('[%(asctime)s] %(levelname)s: %(message)s'))
+        file_handler.setLevel(log_file_log_level)
+        handlers = (console_handler, file_handler)
     else:
         handlers = (console_handler,)
 
-    logging.basicConfig(level=min(console_log_level, syslog_log_level), handlers=handlers)
+    logging.basicConfig(
+        level=min(console_log_level, syslog_log_level, log_file_log_level), handlers=handlers
+    )

docs/how-to/inspect-your-backups.md

@@ -70,6 +70,20 @@ Or to increase syslog logging to include debug spew:
 borgmatic --syslog-verbosity 2
 ```
 
+### Logging to file
+
+If you don't want to use syslog, and you'd rather borgmatic log to a plain
+file, use the `--log-file` flag:
+
+```bash
+borgmatic --log-file /path/to/file.log
+```
+
+Note that if you use the `--log-file` flag, you are responsible for rotating
+the log file so it doesn't grow too large. Also, there is also
+`--log-file-verbosity` flag to customize the log file's log level.
+
+
 ### systemd journal
 
 If your local syslog daemon is systemd's journal, be aware that journald by

docs/how-to/monitor-your-backups.md

@@ -26,12 +26,15 @@ alert. But note that if borgmatic doesn't actually run, this alert won't fire.
 See [error
 hooks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hooks)
 below for how to configure this.
-4. **borgmatic Healthchecks hook**: This feature integrates with the
-[Healthchecks](https://healthchecks.io/) service, and pings Healthchecks
-whenever borgmatic runs. That way, Healthchecks can alert you when something
-goes wrong or it doesn't hear from borgmatic for a configured interval. See
+4. **borgmatic monitoring hooks**: This feature integrates with monitoring
+services like [Healthchecks](https://healthchecks.io/) and
+[Cronitor](https://cronitor.io), and pings these services whenever borgmatic
+runs. That way, you'll receive an alert when something goes wrong or the
+service doesn't hear from borgmatic for a configured interval. See
 [Healthchecks
 hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook)
+and [Cronitor
+hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook)
 below for how to configure this.
 3. **Third-party monitoring software**: You can use traditional monitoring
 software to consume borgmatic JSON output and track when the last
@@ -115,6 +118,27 @@ mechanisms](https://healthchecks.io/#welcome-integrations) when backups fail
 or it doesn't hear from borgmatic for a certain period of time.
 
 
+## Cronitor hook
+
+[Cronitor](https://cronitor.io/) provides "Cron monitoring and uptime healthchecks
+for websites, services and APIs", and borgmatic has built-in
+integration with it. Once you create a Cronitor account and cron job monitor on
+their site, all you need to do is configure borgmatic with the unique "Ping
+API URL" for your monitor. Here's an example:
+
+
+```yaml
+hooks:
+    cronitor: https://cronitor.link/d3x0c1
+```
+
+With this hook in place, borgmatic will ping your Cronitor monitor when a
+backup begins, ends, or errors. Then you can configure Cronitor to notify you
+by a [variety of
+mechanisms](https://cronitor.io/docs/cron-job-notifications) when backups
+fail or it doesn't hear from borgmatic for a certain period of time.
+
+
 ## Scripting borgmatic
 
 To consume the output of borgmatic in other software, you can include an

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.4.2'
+VERSION = '1.4.5'
 
 
 setup(

test_requirements.txt

@@ -1,12 +1,12 @@
 appdirs==1.4.3
 atomicwrites==1.3.0
-attrs==19.1.0
+attrs==19.3.0
 black==19.3b0; python_version >= '3.6'
 click==7.0
 colorama==0.4.1
 coverage==4.5.4
 docopt==0.6.2
-flake8==3.7.8
+flake8==3.7.9
 flexmock==0.10.4
 isort==4.3.21
 mccabe==0.6.1
@@ -16,8 +16,8 @@ py==1.8.0
 pycodestyle==2.5.0
 pyflakes==2.1.1
 pykwalify==1.7.0
-pytest==5.1.2
-pytest-cov==2.7.1
+pytest==5.2.2
+pytest-cov==2.8.1
 python-dateutil==2.8.0
 PyYAML==5.1.2
 requests==2.22.0

tests/unit/borg/test_create.py

@@ -150,6 +150,30 @@ def test_make_exclude_flags_includes_exclude_if_present_when_in_config():
     assert exclude_flags == ('--exclude-if-present', 'exclude_me')
 
 
+def test_make_exclude_flags_includes_keep_exclude_tags_when_true_in_config():
+    exclude_flags = module._make_exclude_flags(location_config={'keep_exclude_tags': True})
+
+    assert exclude_flags == ('--keep-exclude-tags',)
+
+
+def test_make_exclude_flags_does_not_include_keep_exclude_tags_when_false_in_config():
+    exclude_flags = module._make_exclude_flags(location_config={'keep_exclude_tags': False})
+
+    assert exclude_flags == ()
+
+
+def test_make_exclude_flags_includes_exclude_nodump_when_true_in_config():
+    exclude_flags = module._make_exclude_flags(location_config={'exclude_nodump': True})
+
+    assert exclude_flags == ('--exclude-nodump',)
+
+
+def test_make_exclude_flags_does_not_include_exclude_nodump_when_false_in_config():
+    exclude_flags = module._make_exclude_flags(location_config={'exclude_nodump': False})
+
+    assert exclude_flags == ()
+
+
 def test_make_exclude_flags_is_empty_when_config_has_no_excludes():
     exclude_flags = module._make_exclude_flags(location_config={})
 

tests/unit/hooks/test_cronitor.py

@@ -0,0 +1,17 @@
+from flexmock import flexmock
+
+from borgmatic.hooks import cronitor as module
+
+
+def test_ping_cronitor_hits_ping_url():
+    ping_url = 'https://example.com'
+    append = 'failed-so-hard'
+    flexmock(module.requests).should_receive('get').with_args('{}/{}'.format(ping_url, append))
+
+    module.ping_cronitor(ping_url, 'config.yaml', dry_run=False, append=append)
+
+
+def test_ping_cronitor_without_ping_url_does_not_raise():
+    flexmock(module.requests).should_receive('get').never()
+
+    module.ping_cronitor(ping_url=None, config_filename='config.yaml', dry_run=False, append='oops')

tests/unit/test_logger.py

@@ -193,3 +193,33 @@ def test_configure_logging_skips_syslog_if_interactive_console():
     flexmock(module.logging.handlers).should_receive('SysLogHandler').never()
 
     module.configure_logging(console_log_level=logging.INFO)
+
+
+def test_configure_logging_to_logfile_instead_of_syslog():
+    # syslog skipped in non-interactive console if --log-file argument provided
+    flexmock(module).should_receive('interactive_console').and_return(False)
+    flexmock(module.logging).should_receive('basicConfig').with_args(
+        level=logging.DEBUG, handlers=tuple
+    )
+    flexmock(module.os.path).should_receive('exists').with_args('/dev/log').and_return(True)
+    flexmock(module.logging.handlers).should_receive('SysLogHandler').never()
+    file_handler = logging.FileHandler('/tmp/logfile')
+    flexmock(module.logging).should_receive('FileHandler').with_args('/tmp/logfile').and_return(
+        file_handler
+    ).once()
+
+    module.configure_logging(
+        console_log_level=logging.INFO, log_file_log_level=logging.DEBUG, log_file='/tmp/logfile'
+    )
+
+
+def test_configure_logging_skips_logfile_if_argument_is_none():
+    # No FileHandler added if argument --log-file is None
+    flexmock(module).should_receive('interactive_console').and_return(False)
+    flexmock(module.logging).should_receive('basicConfig').with_args(
+        level=logging.INFO, handlers=tuple
+    )
+    flexmock(module.os.path).should_receive('exists').and_return(False)
+    flexmock(module.logging).should_receive('FileHandler').never()
+
+    module.configure_logging(console_log_level=logging.INFO, log_file=None)