Bump version for release.

Merge pull request #49 from jpaniagualaconich/specify-pg-dump-restore-commands

NEWS

@@ -1,3 +1,10 @@
+1.7.5
+ * #311: Override PostgreSQL dump/restore commands via configuration options.
+ * #604: Fix traceback when a configuration section is present but lacking any options.
+ * #607: Clarify documentation examples for include merging and deep merging.
+ * #611: Fix "data" consistency check to support "check_last" and consistency "prefix" options.
+ * #613: Clarify documentation about multiple repositories and separate configuration files.
+
 1.7.4
  * #596: Fix special file detection erroring when broken symlinks are encountered.
  * #597, #598: Fix regression in which "check" action errored on certain systems ("Cannot determine

borgmatic/borg/check.py

@@ -166,6 +166,12 @@ def make_check_flags(local_borg_version, checks, check_last=None, prefix=None):
     "--last" flag. And if a prefix value is given and "archives" is in checks, then include a
     "--match-archives" flag.
     '''
+    if 'data' in checks:
+        data_flags = ('--verify-data',)
+        checks += ('archives',)
+    else:
+        data_flags = ()
+
     if 'archives' in checks:
         last_flags = ('--last', str(check_last)) if check_last else ()
         if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
@@ -176,17 +182,13 @@ def make_check_flags(local_borg_version, checks, check_last=None, prefix=None):
         last_flags = ()
         match_archives_flags = ()
         if check_last:
-            logger.info('Ignoring check_last option, as "archives" is not in consistency checks')
+            logger.warning(
+                'Ignoring check_last option, as "archives" or "data" are not in consistency checks'
+            )
         if prefix:
-            logger.info(
-                'Ignoring consistency prefix option, as "archives" is not in consistency checks'
+            logger.warning(
+                'Ignoring consistency prefix option, as "archives" or "data" are not in consistency checks'
             )
-
-    if 'data' in checks:
-        data_flags = ('--verify-data',)
-        checks += ('archives',)
-    else:
-        data_flags = ()
 
     common_flags = last_flags + match_archives_flags + data_flags
 

borgmatic/config/normalize.py

@@ -8,49 +8,53 @@ def normalize(config_filename, config):
     message warnings produced based on the normalization performed.
     '''
     logs = []
+    location = config.get('location') or {}
+    storage = config.get('storage') or {}
+    consistency = config.get('consistency') or {}
+    hooks = config.get('hooks') or {}
 
     # Upgrade exclude_if_present from a string to a list.
-    exclude_if_present = config.get('location', {}).get('exclude_if_present')
+    exclude_if_present = location.get('exclude_if_present')
     if isinstance(exclude_if_present, str):
         config['location']['exclude_if_present'] = [exclude_if_present]
 
     # Upgrade various monitoring hooks from a string to a dict.
-    healthchecks = config.get('hooks', {}).get('healthchecks')
+    healthchecks = hooks.get('healthchecks')
     if isinstance(healthchecks, str):
         config['hooks']['healthchecks'] = {'ping_url': healthchecks}
 
-    cronitor = config.get('hooks', {}).get('cronitor')
+    cronitor = hooks.get('cronitor')
     if isinstance(cronitor, str):
         config['hooks']['cronitor'] = {'ping_url': cronitor}
 
-    pagerduty = config.get('hooks', {}).get('pagerduty')
+    pagerduty = hooks.get('pagerduty')
     if isinstance(pagerduty, str):
         config['hooks']['pagerduty'] = {'integration_key': pagerduty}
 
-    cronhub = config.get('hooks', {}).get('cronhub')
+    cronhub = hooks.get('cronhub')
     if isinstance(cronhub, str):
         config['hooks']['cronhub'] = {'ping_url': cronhub}
 
     # Upgrade consistency checks from a list of strings to a list of dicts.
-    checks = config.get('consistency', {}).get('checks')
+    checks = consistency.get('checks')
     if isinstance(checks, list) and len(checks) and isinstance(checks[0], str):
         config['consistency']['checks'] = [{'name': check_type} for check_type in checks]
 
     # Rename various configuration options.
-    numeric_owner = config.get('location', {}).pop('numeric_owner', None)
+    numeric_owner = location.pop('numeric_owner', None)
     if numeric_owner is not None:
         config['location']['numeric_ids'] = numeric_owner
 
-    bsd_flags = config.get('location', {}).pop('bsd_flags', None)
+    bsd_flags = location.pop('bsd_flags', None)
     if bsd_flags is not None:
         config['location']['flags'] = bsd_flags
 
-    remote_rate_limit = config.get('storage', {}).pop('remote_rate_limit', None)
+    remote_rate_limit = storage.pop('remote_rate_limit', None)
     if remote_rate_limit is not None:
         config['storage']['upload_rate_limit'] = remote_rate_limit
 
     # Upgrade remote repositories to ssh:// syntax, required in Borg 2.
-    repositories = config.get('location', {}).get('repositories')
+    repositories = location.get('repositories')
     if repositories:
         config['location']['repositories'] = []
         for repository in repositories:

borgmatic/config/schema.yaml

@@ -764,6 +764,32 @@ properties:
                             description: |
                                 Path to a certificate revocation list.
                             example: "/root/.postgresql/root.crl"
+                        pg_dump_command:
+                            type: string
+                            description: |
+                                Command to use instead of "pg_dump" or
+                                "pg_dumpall". This can be used to run a specific
+                                pg_dump version (e.g., one inside a running
+                                docker container). Defaults to "pg_dump" for
+                                single database dump or "pg_dumpall" to dump
+                                all databases.
+                            example: docker exec my_pg_container pg_dump
+                        pg_restore_command:
+                            type: string
+                            description: |
+                                Command to use instead of "pg_restore". This
+                                can be used to run a specific pg_restore
+                                version (e.g., one inside a running docker
+                                container). Defaults to "pg_restore".
+                            example: docker exec my_pg_container pg_restore
+                        psql_command:
+                            type: string
+                            description: |
+                                Command to use instead of "psql". This can be
+                                used to run a specific psql version (e.g.,
+                                one inside a running docker container).
+                                Defaults to "psql".
+                            example: docker exec my_pg_container psql
                         options:
                             type: string
                             description: |

borgmatic/hooks/postgresql.py

@@ -56,13 +56,10 @@ def dump_databases(databases, log_prefix, location_config, dry_run):
         )
         all_databases = bool(name == 'all')
         dump_format = database.get('format', 'custom')
+        default_dump_command = 'pg_dumpall' if all_databases else 'pg_dump'
+        dump_command = database.get('pg_dump_command') or default_dump_command
         command = (
-            (
-                'pg_dumpall' if all_databases else 'pg_dump',
-                '--no-password',
-                '--clean',
-                '--if-exists',
-            )
+            (dump_command, '--no-password', '--clean', '--if-exists',)
             + (('--host', database['hostname']) if 'hostname' in database else ())
             + (('--port', str(database['port'])) if 'port' in database else ())
             + (('--username', database['username']) if 'username' in database else ())
@@ -140,16 +137,18 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
     dump_filename = dump.make_database_dump_filename(
         make_dump_path(location_config), database['name'], database.get('hostname')
     )
+    psql_command = database.get('psql_command') or 'psql'
     analyze_command = (
-        ('psql', '--no-password', '--quiet')
+        (psql_command, '--no-password', '--quiet')
         + (('--host', database['hostname']) if 'hostname' in database else ())
         + (('--port', str(database['port'])) if 'port' in database else ())
         + (('--username', database['username']) if 'username' in database else ())
         + (('--dbname', database['name']) if not all_databases else ())
         + ('--command', 'ANALYZE')
     )
+    pg_restore_command = database.get('pg_restore_command') or 'pg_restore'
     restore_command = (
-        ('psql' if all_databases else 'pg_restore', '--no-password')
+        (psql_command if all_databases else pg_restore_command, '--no-password')
         + (
             ('--if-exists', '--exit-on-error', '--clean', '--dbname', database['name'])
             if not all_databases

docs/how-to/backup-your-databases.md

@@ -220,8 +220,8 @@ such files. Common directories to exclude are `/dev` and `/run`, but that may
 not be exhaustive. <span class="minilink minilink-addedin">New in version
 1.7.3</span> When database hooks are enabled, borgmatic automatically excludes
 special files that may cause Borg to hang, so you no longer need to manually
-exclude them. You can override/prevent this behavior by explicitly setting
-`read_special` to true.
+exclude them. (This includes symlinks with special files as a destination.) You
+can override/prevent this behavior by explicitly setting `read_special` to true.
 
 
 ### Manual restoration

docs/how-to/deal-with-very-large-backups.md

@@ -44,9 +44,11 @@ consistency checks with `check` on a much less frequent basis (e.g. with
 
 ### Consistency check configuration
 
-Another option is to customize your consistency checks. The default
-consistency checks run both full-repository checks and per-archive checks
-within each repository no more than once a month.
+Another option is to customize your consistency checks. By default, if you
+omit consistency checks from configuration, borgmatic runs full-repository
+checks (`repository`) and per-archive checks (`archives`) within each
+repository, no more than once a month. This is equivalent to what `borg check`
+does if run without options.
 
 But if you find that archive checks are too slow, for example, you can
 configure borgmatic to run repository checks only. Configure this in the

docs/how-to/make-backups-redundant.md

@@ -42,3 +42,13 @@ potentially across providers.
 See [Borg repository URLs
 documentation](https://borgbackup.readthedocs.io/en/stable/usage/general.html#repository-urls)
 for more information on how to specify local and remote repository paths.
+
+### Different options per repository
+
+What if you want borgmatic to backup to multiple repositories—while also
+setting different options for each one? In that case, you'll need to use
+[a separate borgmatic configuration file for each
+repository](https://torsion.org/borgmatic/docs/how-to/make-per-application-backups/)
+instead of the multiple repositories in one configuration file as described
+above. That's because all of the repositories in a particular configuration
+file get the same options applied.

docs/how-to/make-per-application-backups.md

@@ -106,11 +106,60 @@ But if you do want to merge in a YAML key *and* its values, keep reading!
 
 ## Include merging
 
-If you need to get even fancier and pull in common configuration options while
-potentially overriding individual options, you can perform a YAML merge of
-included configuration using the YAML `<<` key. For instance, here's an
-example of a main configuration file that pulls in two retention options via
-an include and then overrides one of them locally:
+If you need to get even fancier and merge in common configuration options, you
+can perform a YAML merge of included configuration using the YAML `<<` key.
+For instance, here's an example of a main configuration file that pulls in
+retention and consistency options via a single include:
+
+```yaml
+<<: !include /etc/borgmatic/common.yaml
+
+location:
+   ...
+```
+
+This is what `common.yaml` might look like:
+
+```yaml
+retention:
+    keep_hourly: 24
+    keep_daily: 7
+
+consistency:
+    checks:
+        - name: repository
+```
+
+Once this include gets merged in, the resulting configuration would have all
+of the `location` options from the original configuration file *and* the
+`retention` and `consistency` options from the include.
+
+Prior to borgmatic version 1.6.0, when there's a section collision between the
+local file and the merged include, the local file's section takes precedence.
+So if the `retention` section appears in both the local file and the include
+file, the included `retention` is ignored in favor of the local `retention`.
+But see below about deep merge in version 1.6.0+.
+
+Note that this `<<` include merging syntax is only for merging in mappings
+(configuration options and their values). But if you'd like to include a
+single value directly, please see the section above about standard includes.
+
+Additionally, there is a limitation preventing multiple `<<` include merges
+per section. So for instance, that means you can do one `<<` merge at the
+global level, another `<<` within each configuration section, etc. (This is a
+YAML limitation.)
+
+
+### Deep merge
+
+<span class="minilink minilink-addedin">New in version 1.6.0</span> borgmatic
+performs a deep merge of merged include files, meaning that values are merged
+at all levels in the two configuration files. This allows you to include
+common configuration—up to full borgmatic configuration files—while overriding
+only the parts you want to customize.
+
+For instance, here's an example of a main configuration file that pulls in two
+retention options via an include and then overrides one of them locally:
 
 ```yaml
 <<: !include /etc/borgmatic/common.yaml
@@ -136,24 +185,8 @@ Once this include gets merged in, the resulting configuration would have a
 When there's an option collision between the local file and the merged
 include, the local file's option takes precedence.
 
-Note that this `<<` include merging syntax is only for merging in mappings
-(configuration options and their values). But if you'd like to include a
-single value directly, please see the section above about standard includes.
-
-Additionally, there is a limitation preventing multiple `<<` include merges
-per section. So for instance, that means you can do one `<<` merge at the
-global level, another `<<` within each configuration section, etc. (This is a
-YAML limitation.)
-
-
-### Deep merge
-
-<span class="minilink minilink-addedin">New in version 1.6.0</span> borgmatic
-performs a deep merge of merged include files, meaning that values are merged
-at all levels in the two configuration files. Colliding list values are
-appended together. This allows you to include common configuration—up to full
-borgmatic configuration files—while overriding only the parts you want to
-customize.
+<span class="minilink minilink-addedin">New in version 1.6.1</span> Colliding
+list values are appended together.
 
 
 ## Configuration overrides

docs/how-to/upgrade.md

@@ -160,17 +160,22 @@ Then, run the `rcreate` action (formerly `init`) to create that new Borg 2
 repository:
 
 ```bash
-borgmatic rcreate --verbosity 1 --encryption repokey-aes-ocb \
+borgmatic rcreate --verbosity 1 --encryption repokey-blake2-aes-ocb \
     --source-repository original.borg --repository upgraded.borg
 ```
 
-(Note that `repokey-chacha20-poly1305` may be faster than `repokey-aes-ocb` on
-certain platforms like ARM64.)
-
 This creates an empty repository and doesn't actually transfer any data yet.
 The `--source-repository` flag is necessary to reuse key material from your
 Borg 1 repository so that the subsequent data transfer can work.
 
+The `--encryption` value above selects the same chunk ID algorithm (`blake2`)
+used in Borg 1, thereby making deduplication work across transferred archives
+and new archives. Note that `repokey-blake2-chacha20-poly1305` may be faster
+than `repokey-blake2-aes-ocb` on certain platforms like ARM64. Read about
+[Borg encryption
+modes](https://borgbackup.readthedocs.io/en/2.0.0b3/usage/rcreate.html#encryption-mode-tldr)
+for the menu of available encryption modes.
+
 To transfer data from your original Borg 1 repository to your newly created
 Borg 2 repository:
 
@@ -191,7 +196,7 @@ confirmation of success—or tells you if something hasn't been transferred yet.
 Note that by omitting the `--upgrader` flag, you can also do archive transfers
 between Borg 2 repositories without upgrading, even down to individual
 archives. For more on that functionality, see the [Borg transfer
-documentation](https://borgbackup.readthedocs.io/en/2.0.0b1/usage/transfer.html).
+documentation](https://borgbackup.readthedocs.io/en/2.0.0b3/usage/transfer.html).
 
 That's it! Now you can use your new Borg 2 repository as normal with
 borgmatic. If you've got multiple repositories, repeat the above process for

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.7.4'
+VERSION = '1.7.5'
 
 
 setup(

test_requirements.txt

@@ -14,8 +14,8 @@ py==1.10.0
 pycodestyle==2.8.0
 pyflakes==2.4.0
 jsonschema==3.2.0
-pytest==6.2.5
-pytest-cov==3.0.0
+pytest==7.2.0
+pytest-cov==4.0.0
 regex; python_version >= '3.8'
 requests==2.25.0
 ruamel.yaml>0.15.0,<0.18.0

tests/unit/borg/test_check.py

@@ -265,6 +265,14 @@ def test_make_check_flags_with_archives_check_and_last_includes_last_flag():
     assert flags == ('--archives-only', '--last', '3')
 
 
+def test_make_check_flags_with_data_check_and_last_includes_last_flag():
+    flexmock(module.feature).should_receive('available').and_return(True)
+
+    flags = module.make_check_flags('1.2.3', ('data',), check_last=3)
+
+    assert flags == ('--archives-only', '--last', '3', '--verify-data')
+
+
 def test_make_check_flags_with_repository_check_and_last_omits_last_flag():
     flexmock(module.feature).should_receive('available').and_return(True)
 
@@ -289,6 +297,14 @@ def test_make_check_flags_with_archives_check_and_prefix_includes_match_archives
     assert flags == ('--archives-only', '--match-archives', 'sh:foo-*')
 
 
+def test_make_check_flags_with_data_check_and_prefix_includes_match_archives_flag():
+    flexmock(module.feature).should_receive('available').and_return(True)
+
+    flags = module.make_check_flags('1.2.3', ('data',), prefix='foo-')
+
+    assert flags == ('--archives-only', '--match-archives', 'sh:foo-*', '--verify-data')
+
+
 def test_make_check_flags_with_archives_check_and_empty_prefix_omits_match_archives_flag():
     flexmock(module.feature).should_receive('available').and_return(True)
 

tests/unit/config/test_normalize.py

@@ -21,11 +21,13 @@ from borgmatic.config import normalize as module
             {'location': {'source_directories': ['foo', 'bar']}},
             False,
         ),
+        ({'location': None}, {'location': None}, False,),
         (
             {'storage': {'compression': 'yes_please'}},
             {'storage': {'compression': 'yes_please'}},
             False,
         ),
+        ({'storage': None}, {'storage': None}, False,),
         (
             {'hooks': {'healthchecks': 'https://example.com'}},
             {'hooks': {'healthchecks': {'ping_url': 'https://example.com'}}},
@@ -46,11 +48,18 @@ from borgmatic.config import normalize as module
             {'hooks': {'cronhub': {'ping_url': 'https://example.com'}}},
             False,
         ),
+        ({'hooks': None}, {'hooks': None}, False,),
         (
             {'consistency': {'checks': ['archives']}},
             {'consistency': {'checks': [{'name': 'archives'}]}},
             False,
         ),
+        (
+            {'consistency': {'checks': ['archives']}},
+            {'consistency': {'checks': [{'name': 'archives'}]}},
+            False,
+        ),
+        ({'consistency': None}, {'consistency': None}, False,),
         ({'location': {'numeric_owner': False}}, {'location': {'numeric_ids': False}}, False,),
         ({'location': {'bsd_flags': False}}, {'location': {'flags': False}}, False,),
         (

tests/unit/hooks/test_postgresql.py

@@ -223,6 +223,36 @@ def test_dump_databases_runs_pg_dumpall_for_all_databases():
     assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
 
 
+def test_dump_databases_runs_non_default_pg_dump():
+    databases = [{'name': 'foo', 'pg_dump_command': 'special_pg_dump'}]
+    process = flexmock()
+    flexmock(module).should_receive('make_dump_path').and_return('')
+    flexmock(module.dump).should_receive('make_database_dump_filename').and_return(
+        'databases/localhost/foo'
+    )
+    flexmock(module.dump).should_receive('create_named_pipe_for_dump')
+    flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
+
+    flexmock(module).should_receive('execute_command').with_args(
+        (
+            'special_pg_dump',
+            '--no-password',
+            '--clean',
+            '--if-exists',
+            '--format',
+            'custom',
+            'foo',
+            '>',
+            'databases/localhost/foo',
+        ),
+        shell=True,
+        extra_environment={'PGSSLMODE': 'disable'},
+        run_to_completion=False,
+    ).and_return(process).once()
+
+    assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
+
+
 def test_restore_database_dump_runs_pg_restore():
     database_config = [{'name': 'foo'}]
     extract_process = flexmock(stdout=flexmock())
@@ -388,6 +418,40 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
     )
 
 
+def test_restore_database_dump_runs_non_default_pg_restore_and_psql():
+    database_config = [
+        {'name': 'foo', 'pg_restore_command': 'special_pg_restore', 'psql_command': 'special_psql'}
+    ]
+    extract_process = flexmock(stdout=flexmock())
+
+    flexmock(module).should_receive('make_dump_path')
+    flexmock(module.dump).should_receive('make_database_dump_filename')
+    flexmock(module).should_receive('make_extra_environment').and_return({'PGSSLMODE': 'disable'})
+    flexmock(module).should_receive('execute_command_with_processes').with_args(
+        (
+            'special_pg_restore',
+            '--no-password',
+            '--if-exists',
+            '--exit-on-error',
+            '--clean',
+            '--dbname',
+            'foo',
+        ),
+        processes=[extract_process],
+        output_log_level=logging.DEBUG,
+        input_file=extract_process.stdout,
+        extra_environment={'PGSSLMODE': 'disable'},
+    ).once()
+    flexmock(module).should_receive('execute_command').with_args(
+        ('special_psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'),
+        extra_environment={'PGSSLMODE': 'disable'},
+    ).once()
+
+    module.restore_database_dump(
+        database_config, 'test.yaml', {}, dry_run=False, extract_process=extract_process
+    )
+
+
 def test_restore_database_dump_with_dry_run_skips_restore():
     database_config = [{'name': 'foo'}]