Support for Borg --keep-exclude-tags and --exclude-nodump options (#234).

NEWS

@@ -1,3 +1,15 @@
+1.4.4
+ * #234: Support for Borg --keep-exclude-tags and --exclude-nodump options.
+
+1.4.3
+ * Monitor backups with Cronitor hook integration. See the documentation for more information:
+   https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook
+
+1.4.2
+ * Extract files to a particular directory via "borgmatic extract --destination" flag.
+ * Rename "borgmatic extract --restore-path" flag to "--path" to reduce confusion with the separate
+   "borgmatic restore" action. Any uses of "--restore-path" will continue working.
+
 1.4.1
  * #229: Restore backed up PostgreSQL databases via "borgmatic restore" action. See the
    documentation for more information:

borgmatic/borg/create.py

@@ -90,8 +90,18 @@ def _make_exclude_flags(location_config, exclude_filename=None):
     caches_flag = ('--exclude-caches',) if location_config.get('exclude_caches') else ()
     if_present = location_config.get('exclude_if_present')
     if_present_flags = ('--exclude-if-present', if_present) if if_present else ()
+    keep_exclude_tags_flags = (
+        ('--keep-exclude-tags',) if location_config.get('keep_exclude_tags') else ()
+    )
+    exclude_nodump_flags = ('--exclude-nodump',) if location_config.get('exclude_nodump') else ()
 
-    return exclude_from_flags + caches_flag + if_present_flags
+    return (
+        exclude_from_flags
+        + caches_flag
+        + if_present_flags
+        + keep_exclude_tags_flags
+        + exclude_nodump_flags
+    )
 
 
 BORGMATIC_SOURCE_DIRECTORY = '~/.borgmatic'

borgmatic/borg/extract.py

@@ -55,7 +55,7 @@ def extract_archive(
     dry_run,
     repository,
     archive,
-    restore_paths,
+    paths,
     location_config,
     storage_config,
     local_path='borg',
@@ -83,7 +83,7 @@ def extract_archive(
         + (('--dry-run',) if dry_run else ())
         + (('--progress',) if progress else ())
         + ('::'.join((os.path.abspath(repository), archive)),)
-        + (tuple(restore_paths) if restore_paths else ())
+        + (tuple(paths) if paths else ())
     )
 
     # The progress output isn't compatible with captured and logged output, as progress messes with

borgmatic/commands/arguments.py

@@ -273,13 +273,20 @@ def parse_arguments(*unparsed_arguments):
         '--repository',
         help='Path of repository to extract, defaults to the configured repository if there is only one',
     )
-    extract_group.add_argument('--archive', help='Name of archive to operate on', required=True)
+    extract_group.add_argument('--archive', help='Name of archive to extract', required=True)
     extract_group.add_argument(
+        '--path',
         '--restore-path',
         metavar='PATH',
         nargs='+',
-        dest='restore_paths',
-        help='Paths to restore from archive, defaults to the entire archive',
+        dest='paths',
+        help='Paths to extract from archive, defaults to the entire archive',
+    )
+    extract_group.add_argument(
+        '--destination',
+        metavar='PATH',
+        dest='destination',
+        help='Directory to extract files into, defaults to the current directory',
     )
     extract_group.add_argument(
         '--progress',
@@ -304,7 +311,7 @@ def parse_arguments(*unparsed_arguments):
         '--repository',
         help='Path of repository to restore from, defaults to the configured repository if there is only one',
     )
-    restore_group.add_argument('--archive', help='Name of archive to operate on', required=True)
+    restore_group.add_argument('--archive', help='Name of archive to restore from', required=True)
     restore_group.add_argument(
         '--database',
         metavar='NAME',

borgmatic/commands/borgmatic.py

@@ -18,7 +18,7 @@ from borgmatic.borg import list as borg_list
 from borgmatic.borg import prune as borg_prune
 from borgmatic.commands.arguments import parse_arguments
 from borgmatic.config import checks, collect, convert, validate
-from borgmatic.hooks import command, healthchecks, postgresql
+from borgmatic.hooks import command, cronitor, healthchecks, postgresql
 from borgmatic.logger import configure_logging, should_do_markup
 from borgmatic.signals import configure_signals
 from borgmatic.verbosity import verbosity_to_log_level
@@ -56,6 +56,9 @@ def run_configuration(config_filename, config, arguments):
             healthchecks.ping_healthchecks(
                 hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'start'
             )
+            cronitor.ping_cronitor(
+                hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'run'
+            )
             command.execute_hook(
                 hooks.get('before_backup'),
                 hooks.get('umask'),
@@ -108,6 +111,9 @@ def run_configuration(config_filename, config, arguments):
             healthchecks.ping_healthchecks(
                 hooks.get('healthchecks'), config_filename, global_arguments.dry_run
             )
+            cronitor.ping_cronitor(
+                hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'complete'
+            )
         except (OSError, CalledProcessError) as error:
             encountered_error = error
             yield from make_error_log_records(
@@ -129,6 +135,9 @@ def run_configuration(config_filename, config, arguments):
             healthchecks.ping_healthchecks(
                 hooks.get('healthchecks'), config_filename, global_arguments.dry_run, 'fail'
             )
+            cronitor.ping_cronitor(
+                hooks.get('cronitor'), config_filename, global_arguments.dry_run, 'fail'
+            )
         except (OSError, CalledProcessError) as error:
             yield from make_error_log_records(
                 '{}: Error running on-error hook'.format(config_filename), error
@@ -215,12 +224,12 @@ def run_actions(
                 global_arguments.dry_run,
                 repository,
                 arguments['extract'].archive,
-                arguments['extract'].restore_paths,
+                arguments['extract'].paths,
                 location,
                 storage,
                 local_path=local_path,
                 remote_path=remote_path,
-                destination_path=None,
+                destination_path=arguments['extract'].destination,
                 progress=arguments['extract'].progress,
             )
     if 'restore' in arguments:

borgmatic/config/schema.yaml

@@ -126,6 +126,17 @@ map:
                     Exclude directories that contain a file with the given filename. Defaults to not
                     set.
                 example: .nobackup
+            keep_exclude_tags:
+                type: bool
+                desc: |
+                    If true, the exclude_if_present filename is included in backups. Defaults to
+                    false, meaning that the exclude_if_present filename is omitted from backups.
+                example: true
+            exclude_nodump:
+                type: bool
+                desc: |
+                    Exclude files with the NODUMP flag. Defaults to false.
+                example: true
     storage:
         desc: |
             Repository storage options. See
@@ -430,6 +441,13 @@ map:
                     Create an account at https://healthchecks.io if you'd like to use this service.
                 example:
                     https://hc-ping.com/your-uuid-here
+            cronitor:
+                type: str
+                desc: |
+                    Cronitor ping URL to notify when a backup begins, ends, or errors. Create an
+                    account at https://cronitor.io if you'd like to use this service.
+                example:
+                    https://cronitor.link/d3x0c1
             before_everything:
                 seq:
                     - type: str

borgmatic/hooks/cronitor.py

@@ -0,0 +1,24 @@
+import logging
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+def ping_cronitor(ping_url, config_filename, dry_run, append):
+    '''
+    Ping the given Cronitor URL, appending the append string. Use the given configuration filename
+    in any log entries. If this is a dry run, then don't actually ping anything.
+    '''
+    if not ping_url:
+        logger.debug('{}: No Cronitor hook set'.format(config_filename))
+        return
+
+    dry_run_label = ' (dry run; not actually pinging)' if dry_run else ''
+    ping_url = '{}/{}'.format(ping_url, append)
+
+    logger.info('{}: Pinging Cronitor {}{}'.format(config_filename, append, dry_run_label))
+    logger.debug('{}: Using Cronitor ping URL {}'.format(config_filename, ping_url))
+
+    logging.getLogger('urllib3').setLevel(logging.ERROR)
+    requests.get(ping_url)

borgmatic/hooks/healthchecks.py

@@ -7,12 +7,12 @@ logger = logging.getLogger(__name__)
 
 def ping_healthchecks(ping_url_or_uuid, config_filename, dry_run, append=None):
     '''
-    Ping the given healthchecks.io URL or UUID, appending the append string if any. Use the given
+    Ping the given Healthchecks URL or UUID, appending the append string if any. Use the given
     configuration filename in any log entries. If this is a dry run, then don't actually ping
     anything.
     '''
     if not ping_url_or_uuid:
-        logger.debug('{}: No healthchecks hook set'.format(config_filename))
+        logger.debug('{}: No Healthchecks hook set'.format(config_filename))
         return
 
     ping_url = (
@@ -26,11 +26,11 @@ def ping_healthchecks(ping_url_or_uuid, config_filename, dry_run, append=None):
         ping_url = '{}/{}'.format(ping_url, append)
 
     logger.info(
-        '{}: Pinging healthchecks.io{}{}'.format(
+        '{}: Pinging Healthchecks{}{}'.format(
             config_filename, ' ' + append if append else '', dry_run_label
         )
     )
-    logger.debug('{}: Using healthchecks.io ping URL {}'.format(config_filename, ping_url))
+    logger.debug('{}: Using Healthchecks ping URL {}'.format(config_filename, ping_url))
 
     logging.getLogger('urllib3').setLevel(logging.ERROR)
     requests.get(ping_url)

docs/how-to/extract-a-backup.md

@@ -51,11 +51,11 @@ borgmatic extract --repository repo.borg --archive host-2019-...
 ## Extract particular files
 
 Sometimes, you want to extract a single deleted file, rather than extracting
-everything from an archive. To do that, tack on one or more `--restore-path`
-values. For instance:
+everything from an archive. To do that, tack on one or more `--path` values.
+For instance:
 
 ```bash
-borgmatic extract --archive host-2019-... --restore-path path/1 path/2
+borgmatic extract --archive host-2019-... --path path/1 path/2
 ```
 
 Note that the specified restore paths should not have a leading slash. Like a
@@ -63,6 +63,19 @@ whole-archive extract, this also extracts into the current directory. So for
 example, if you happen to be in the directory `/var` and you run the `extract`
 command above, borgmatic will extract `/var/path/1` and `/var/path/2`.
 
+## Extract to a particular destination
+
+By default, borgmatic extracts files into the current directory. To instead
+extract files to a particular destination directory, use the `--destination`
+flag:
+
+```bash
+borgmatic extract --archive host-2019-... --destination /tmp
+```
+
+When using the `--destination` flag, be careful not to overwrite your system's
+files with extracted files unless that is your intent.
+
 
 ## Database restoration
 

docs/how-to/monitor-your-backups.md

@@ -26,12 +26,15 @@ alert. But note that if borgmatic doesn't actually run, this alert won't fire.
 See [error
 hooks](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#error-hooks)
 below for how to configure this.
-4. **borgmatic Healthchecks hook**: This feature integrates with the
-[Healthchecks](https://healthchecks.io/) service, and pings Healthchecks
-whenever borgmatic runs. That way, Healthchecks can alert you when something
-goes wrong or it doesn't hear from borgmatic for a configured interval. See
+4. **borgmatic monitoring hooks**: This feature integrates with monitoring
+services like [Healthchecks](https://healthchecks.io/) and
+[Cronitor](https://cronitor.io), and pings these services whenever borgmatic
+runs. That way, you'll receive an alert when something goes wrong or the
+service doesn't hear from borgmatic for a configured interval. See
 [Healthchecks
 hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#healthchecks-hook)
+and [Cronitor
+hook](https://torsion.org/borgmatic/docs/how-to/monitor-your-backups/#cronitor-hook)
 below for how to configure this.
 3. **Third-party monitoring software**: You can use traditional monitoring
 software to consume borgmatic JSON output and track when the last
@@ -115,6 +118,27 @@ mechanisms](https://healthchecks.io/#welcome-integrations) when backups fail
 or it doesn't hear from borgmatic for a certain period of time.
 
 
+## Cronitor hook
+
+[Cronitor](https://cronitor.io/) provides "Cron monitoring and uptime healthchecks
+for websites, services and APIs", and borgmatic has built-in
+integration with it. Once you create a Cronitor account and cron job monitor on
+their site, all you need to do is configure borgmatic with the unique "Ping
+API URL" for your monitor. Here's an example:
+
+
+```yaml
+hooks:
+    cronitor: https://cronitor.link/d3x0c1
+```
+
+With this hook in place, borgmatic will ping your Cronitor monitor when a
+backup begins, ends, or errors. Then you can configure Cronitor to notify you
+by a [variety of
+mechanisms](https://cronitor.io/docs/cron-job-notifications) when backups
+fail or it doesn't hear from borgmatic for a certain period of time.
+
+
 ## Scripting borgmatic
 
 To consume the output of borgmatic in other software, you can include an

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.4.1'
+VERSION = '1.4.4'
 
 
 setup(

test_requirements.txt

@@ -1,12 +1,12 @@
 appdirs==1.4.3
 atomicwrites==1.3.0
-attrs==19.1.0
+attrs==19.3.0
 black==19.3b0; python_version >= '3.6'
 click==7.0
 colorama==0.4.1
 coverage==4.5.4
 docopt==0.6.2
-flake8==3.7.8
+flake8==3.7.9
 flexmock==0.10.4
 isort==4.3.21
 mccabe==0.6.1
@@ -16,8 +16,8 @@ py==1.8.0
 pycodestyle==2.5.0
 pyflakes==2.1.1
 pykwalify==1.7.0
-pytest==5.1.2
-pytest-cov==2.7.1
+pytest==5.2.2
+pytest-cov==2.8.1
 python-dateutil==2.8.0
 PyYAML==5.1.2
 requests==2.22.0

tests/integration/commands/test_arguments.py

@@ -267,11 +267,11 @@ def test_parse_arguments_disallows_archive_without_extract_restore_or_list():
         module.parse_arguments('--config', 'myconfig', '--archive', 'test')
 
 
-def test_parse_arguments_disallows_restore_paths_without_extract():
+def test_parse_arguments_disallows_paths_without_extract():
     flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
 
     with pytest.raises(SystemExit):
-        module.parse_arguments('--config', 'myconfig', '--restore-path', 'test')
+        module.parse_arguments('--config', 'myconfig', '--path', 'test')
 
 
 def test_parse_arguments_allows_archive_with_extract():

tests/unit/borg/test_create.py

@@ -150,6 +150,30 @@ def test_make_exclude_flags_includes_exclude_if_present_when_in_config():
     assert exclude_flags == ('--exclude-if-present', 'exclude_me')
 
 
+def test_make_exclude_flags_includes_keep_exclude_tags_when_true_in_config():
+    exclude_flags = module._make_exclude_flags(location_config={'keep_exclude_tags': True})
+
+    assert exclude_flags == ('--keep-exclude-tags',)
+
+
+def test_make_exclude_flags_does_not_include_keep_exclude_tags_when_false_in_config():
+    exclude_flags = module._make_exclude_flags(location_config={'keep_exclude_tags': False})
+
+    assert exclude_flags == ()
+
+
+def test_make_exclude_flags_includes_exclude_nodump_when_true_in_config():
+    exclude_flags = module._make_exclude_flags(location_config={'exclude_nodump': True})
+
+    assert exclude_flags == ('--exclude-nodump',)
+
+
+def test_make_exclude_flags_does_not_include_exclude_nodump_when_false_in_config():
+    exclude_flags = module._make_exclude_flags(location_config={'exclude_nodump': False})
+
+    assert exclude_flags == ()
+
+
 def test_make_exclude_flags_is_empty_when_config_has_no_excludes():
     exclude_flags = module._make_exclude_flags(location_config={})
 

tests/unit/borg/test_extract.py

@@ -95,7 +95,7 @@ def test_extract_archive_calls_borg_with_restore_path_parameters():
         dry_run=False,
         repository='repo',
         archive='archive',
-        restore_paths=['path1', 'path2'],
+        paths=['path1', 'path2'],
         location_config={},
         storage_config={},
     )
@@ -109,7 +109,7 @@ def test_extract_archive_calls_borg_with_remote_path_parameters():
         dry_run=False,
         repository='repo',
         archive='archive',
-        restore_paths=None,
+        paths=None,
         location_config={},
         storage_config={},
         remote_path='borg1',
@@ -124,7 +124,7 @@ def test_extract_archive_calls_borg_with_numeric_owner_parameter():
         dry_run=False,
         repository='repo',
         archive='archive',
-        restore_paths=None,
+        paths=None,
         location_config={'numeric_owner': True},
         storage_config={},
     )
@@ -138,7 +138,7 @@ def test_extract_archive_calls_borg_with_umask_parameters():
         dry_run=False,
         repository='repo',
         archive='archive',
-        restore_paths=None,
+        paths=None,
         location_config={},
         storage_config={'umask': '0770'},
     )
@@ -152,7 +152,7 @@ def test_extract_archive_calls_borg_with_lock_wait_parameters():
         dry_run=False,
         repository='repo',
         archive='archive',
-        restore_paths=None,
+        paths=None,
         location_config={},
         storage_config={'lock_wait': '5'},
     )
@@ -167,7 +167,7 @@ def test_extract_archive_with_log_info_calls_borg_with_info_parameter():
         dry_run=False,
         repository='repo',
         archive='archive',
-        restore_paths=None,
+        paths=None,
         location_config={},
         storage_config={},
     )
@@ -184,7 +184,7 @@ def test_extract_archive_with_log_debug_calls_borg_with_debug_parameters():
         dry_run=False,
         repository='repo',
         archive='archive',
-        restore_paths=None,
+        paths=None,
         location_config={},
         storage_config={},
     )
@@ -198,7 +198,7 @@ def test_extract_archive_calls_borg_with_dry_run_parameter():
         dry_run=True,
         repository='repo',
         archive='archive',
-        restore_paths=None,
+        paths=None,
         location_config={},
         storage_config={},
     )
@@ -212,7 +212,7 @@ def test_extract_archive_calls_borg_with_destination_path():
         dry_run=False,
         repository='repo',
         archive='archive',
-        restore_paths=None,
+        paths=None,
         location_config={},
         storage_config={},
         destination_path='/dest',
@@ -231,7 +231,7 @@ def test_extract_archive_calls_borg_with_progress_parameter():
         dry_run=False,
         repository='repo',
         archive='archive',
-        restore_paths=None,
+        paths=None,
         location_config={},
         storage_config={},
         progress=True,

tests/unit/hooks/test_cronitor.py

@@ -0,0 +1,17 @@
+from flexmock import flexmock
+
+from borgmatic.hooks import cronitor as module
+
+
+def test_ping_cronitor_hits_ping_url():
+    ping_url = 'https://example.com'
+    append = 'failed-so-hard'
+    flexmock(module.requests).should_receive('get').with_args('{}/{}'.format(ping_url, append))
+
+    module.ping_cronitor(ping_url, 'config.yaml', dry_run=False, append=append)
+
+
+def test_ping_cronitor_without_ping_url_does_not_raise():
+    flexmock(module.requests).should_receive('get').never()
+
+    module.ping_cronitor(ping_url=None, config_filename='config.yaml', dry_run=False, append='oops')