Bump version for release.

NEWS

@@ -1,3 +1,10 @@
+1.6.6
+ * #559: Update documentation about configuring multiple consistency checks or multiple databases.
+ * #560: Fix all database hooks to error when the requested database to restore isn't present in the
+   Borg archive.
+ * #561: Fix command-line "--override" flag to continue supporting old configuration file formats.
+ * #563: Fix traceback with "create" action and "--json" flag when a database hook is configured.
+
 1.6.5
  * #553: Fix logging to include the full traceback when Borg experiences an internal error, not just
    the first few lines.

borgmatic/borg/extract.py

@@ -137,8 +137,8 @@ def extract_archive(
             extra_environment=borg_environment,
         )
 
-    # Don't give Borg local path, so as to error on warnings, as Borg only gives a warning if the
-    # restore paths don't exist in the archive!
+    # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
+    # if the restore paths don't exist in the archive.
     execute_command(
         full_command, working_directory=destination_path, extra_environment=borg_environment
     )

borgmatic/config/validate.py

@@ -98,8 +98,8 @@ def parse_configuration(config_filename, schema_filename, overrides=None, resolv
     except (ruamel.yaml.error.YAMLError, RecursionError) as error:
         raise Validation_error(config_filename, (str(error),))
 
-    normalize.normalize(config)
     override.apply_overrides(config, overrides)
+    normalize.normalize(config)
     if resolve_env:
         environment.resolve_env_variables(config)
 

borgmatic/execute.py

@@ -51,6 +51,9 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
     process with the requested log level. Additionally, raise a CalledProcessError if a process
     exits with an error (or a warning for exit code 1, if that process matches the Borg local path).
 
+    If output log level is None, then instead of logging, capture output for each process and return
+    it as a dict from the process to its output.
+
     For simplicity, it's assumed that the output buffer for each process is its stdout. But if any
     stdouts are given to exclude, then for any matching processes, log from their stderr instead.
 
@@ -65,6 +68,7 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
         if process.stdout or process.stderr
     }
     output_buffers = list(process_for_output_buffer.keys())
+    captured_outputs = collections.defaultdict(list)
 
     # Log output for each process until they all exit.
     while True:
@@ -99,7 +103,10 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
                     if len(last_lines) > ERROR_OUTPUT_MAX_LINE_COUNT:
                         last_lines.pop(0)
 
-                    logger.log(output_log_level, line)
+                    if output_log_level is None:
+                        captured_outputs[ready_process].append(line)
+                    else:
+                        logger.log(output_log_level, line)
 
         still_running = False
 
@@ -133,6 +140,11 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
         if not still_running:
             break
 
+    if captured_outputs:
+        return {
+            process: '\n'.join(output_lines) for process, output_lines in captured_outputs.items()
+        }
+
 
 def log_command(full_command, input_file, output_file):
     '''
@@ -222,13 +234,14 @@ def execute_command_with_processes(
     run as well. This is useful, for instance, for processes that are streaming output to a named
     pipe that the given command is consuming from.
 
-    If an open output file object is given, then write stdout to the file and only log stderr (but
-    only if an output log level is set). If an open input file object is given, then read stdin from
-    the file.  If shell is True, execute the command within a shell. If an extra environment dict is
-    given, then use it to augment the current environment, and pass the result into the command. If
-    a working directory is given, use that as the present working directory when running the
-    command. If a Borg local path is given, then for any matching command or process (regardless of
-    arguments), treat exit code 1 as a warning instead of an error.
+    If an open output file object is given, then write stdout to the file and only log stderr. But
+    if output log level is None, instead suppress logging and return the captured output for (only)
+    the given command. If an open input file object is given, then read stdin from the file. If
+    shell is True, execute the command within a shell. If an extra environment dict is given, then
+    use it to augment the current environment, and pass the result into the command. If a working
+    directory is given, use that as the present working directory when running the command. If a
+    Borg local path is given, then for any matching command or process (regardless of arguments),
+    treat exit code 1 as a warning instead of an error.
 
     Raise subprocesses.CalledProcessError if an error occurs while running the command or in the
     upstream process.
@@ -259,9 +272,12 @@ def execute_command_with_processes(
                 process.kill()
         raise
 
-    log_outputs(
+    captured_outputs = log_outputs(
         tuple(processes) + (command_process,),
         (input_file, output_file),
         output_log_level,
         borg_local_path=borg_local_path,
     )
+
+    if output_log_level is None:
+        return captured_outputs.get(command_process)

borgmatic/hooks/mongodb.py

@@ -131,12 +131,13 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
     if dry_run:
         return
 
+    # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
+    # if the restore paths don't exist in the archive.
     execute_command_with_processes(
         restore_command,
         [extract_process] if extract_process else [],
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout if extract_process else None,
-        borg_local_path=location_config.get('local_path', 'borg'),
     )
 
 

borgmatic/hooks/mysql.py

@@ -166,11 +166,12 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
     if dry_run:
         return
 
+    # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
+    # if the restore paths don't exist in the archive.
     execute_command_with_processes(
         restore_command,
         [extract_process],
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
         extra_environment=extra_environment,
-        borg_local_path=location_config.get('local_path', 'borg'),
     )

borgmatic/hooks/postgresql.py

@@ -168,12 +168,13 @@ def restore_database_dump(database_config, log_prefix, location_config, dry_run,
     if dry_run:
         return
 
+    # Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
+    # if the restore paths don't exist in the archive.
     execute_command_with_processes(
         restore_command,
         [extract_process] if extract_process else [],
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout if extract_process else None,
         extra_environment=extra_environment,
-        borg_local_path=location_config.get('local_path', 'borg'),
     )
     execute_command(analyze_command, extra_environment=extra_environment)

docs/how-to/backup-your-databases.md

@@ -52,6 +52,8 @@ hooks:
     postgresql_databases:
         - name: users
           hostname: database1.example.org
+        - name: orders
+          hostname: database2.example.org
           port: 5433
           username: postgres
           password: trustsome1
@@ -59,14 +61,14 @@ hooks:
           options: "--role=someone"
     mysql_databases:
         - name: posts
-          hostname: database2.example.org
+          hostname: database3.example.org
           port: 3307
           username: root
           password: trustsome1
           options: "--skip-comments"
     mongodb_databases:
         - name: messages
-          hostname: database3.example.org
+          hostname: database4.example.org
           port: 27018
           username: dbuser
           password: trustsome1

docs/how-to/deal-with-very-large-backups.md

@@ -83,12 +83,15 @@ consistency:
     checks:
         - name: repository
           frequency: 2 weeks
+        - name: archives
+          frequency: 1 month
 ```
 
-This tells borgmatic to run this consistency check at most once every two
-weeks for a given repository. The `frequency` value is a number followed by a
-unit of time, e.g. "3 days", "1 week", "2 months", etc. The `frequency`
-defaults to "always", which means run this check every time checks run.
+This tells borgmatic to run the `repository` consistency check at most once
+every two weeks for a given repository and the `archives` check at most once a
+month. The `frequency` value is a number followed by a unit of time, e.g. "3
+days", "1 week", "2 months", etc. The `frequency` defaults to `always`, which
+means run this check every time checks run.
 
 Unlike a real scheduler like cron, borgmatic only makes a best effort to run
 checks on the configured frequency. It compares that frequency with how long
@@ -97,6 +100,10 @@ within `~/.borgmatic/checks`). If it hasn't been long enough, the check is
 skipped. And you still have to run `borgmatic check` (or just `borgmatic`) in
 order for checks to run, even when a `frequency` is configured!
 
+This also applies *across* configuration files that have the same repository
+configured. Just make sure you have the same check frequency configured in
+each—or the most frequently configured check will apply.
+
 If you want to temporarily ignore your configured frequencies, you can invoke
 `borgmatic check --force` to run checks unconditionally.
 

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.6.5'
+VERSION = '1.6.6'
 
 
 setup(

tests/end-to-end/test_override.py

@@ -0,0 +1,58 @@
+import os
+import shutil
+import subprocess
+import tempfile
+
+
+def generate_configuration(config_path, repository_path):
+    '''
+    Generate borgmatic configuration into a file at the config path, and update the defaults so as
+    to work for testing (including injecting the given repository path and tacking on an encryption
+    passphrase).
+    '''
+    subprocess.check_call(
+        'generate-borgmatic-config --destination {}'.format(config_path).split(' ')
+    )
+    config = (
+        open(config_path)
+        .read()
+        .replace('user@backupserver:sourcehostname.borg', repository_path)
+        .replace('- user@backupserver:{fqdn}', '')
+        .replace('- /home/user/path with spaces', '')
+        .replace('- /home', '- {}'.format(config_path))
+        .replace('- /etc', '')
+        .replace('- /var/log/syslog*', '')
+        + 'storage:\n    encryption_passphrase: "test"'
+    )
+    config_file = open(config_path, 'w')
+    config_file.write(config)
+    config_file.close()
+
+
+def test_override_get_normalized():
+    temporary_directory = tempfile.mkdtemp()
+    repository_path = os.path.join(temporary_directory, 'test.borg')
+    extract_path = os.path.join(temporary_directory, 'extract')
+
+    original_working_directory = os.getcwd()
+    os.mkdir(extract_path)
+    os.chdir(extract_path)
+
+    try:
+        config_path = os.path.join(temporary_directory, 'test.yaml')
+        generate_configuration(config_path, repository_path)
+
+        subprocess.check_call(
+            f'borgmatic -v 2 --config {config_path} init --encryption repokey'.split(' ')
+        )
+
+        # Run borgmatic with an override structured for an outdated config file format. If
+        # normalization is working, it should get normalized and shouldn't error.
+        subprocess.check_call(
+            f'borgmatic create --config {config_path} --override hooks.healthchecks=http://localhost:8888/someuuid'.split(
+                ' '
+            )
+        )
+    finally:
+        os.chdir(original_working_directory)
+        shutil.rmtree(temporary_directory)

tests/integration/test_execute.py

@@ -54,6 +54,30 @@ def test_log_outputs_skips_logs_for_process_with_none_stdout():
     )
 
 
+def test_log_outputs_returns_output_without_logging_for_output_log_level_none():
+    flexmock(module.logger).should_receive('log').never()
+    flexmock(module).should_receive('exit_code_indicates_error').and_return(False)
+
+    hi_process = subprocess.Popen(['echo', 'hi'], stdout=subprocess.PIPE)
+    flexmock(module).should_receive('output_buffer_for_process').with_args(
+        hi_process, ()
+    ).and_return(hi_process.stdout)
+
+    there_process = subprocess.Popen(['echo', 'there'], stdout=subprocess.PIPE)
+    flexmock(module).should_receive('output_buffer_for_process').with_args(
+        there_process, ()
+    ).and_return(there_process.stdout)
+
+    captured_outputs = module.log_outputs(
+        (hi_process, there_process),
+        exclude_stdouts=(),
+        output_log_level=None,
+        borg_local_path='borg',
+    )
+
+    assert captured_outputs == {hi_process: 'hi', there_process: 'there'}
+
+
 def test_log_outputs_includes_error_output_in_exception():
     flexmock(module.logger).should_receive('log')
     flexmock(module).should_receive('exit_code_indicates_error').and_return(True)

tests/unit/hooks/test_mongodb.py

@@ -159,7 +159,7 @@ def test_dump_databases_runs_mongodumpall_for_all_databases():
     assert module.dump_databases(databases, 'test.yaml', {}, dry_run=False) == [process]
 
 
-def test_restore_database_dump_runs_pg_restore():
+def test_restore_database_dump_runs_mongorestore():
     database_config = [{'name': 'foo'}]
     extract_process = flexmock(stdout=flexmock())
 
@@ -170,7 +170,6 @@ def test_restore_database_dump_runs_pg_restore():
         processes=[extract_process],
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
-        borg_local_path='borg',
     ).once()
 
     module.restore_database_dump(
@@ -192,7 +191,7 @@ def test_restore_database_dump_errors_on_multiple_database_config():
         )
 
 
-def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
+def test_restore_database_dump_runs_mongorestore_with_hostname_and_port():
     database_config = [{'name': 'foo', 'hostname': 'database.example.org', 'port': 5433}]
     extract_process = flexmock(stdout=flexmock())
 
@@ -213,7 +212,6 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
         processes=[extract_process],
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
-        borg_local_path='borg',
     ).once()
 
     module.restore_database_dump(
@@ -221,7 +219,7 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
     )
 
 
-def test_restore_database_dump_runs_pg_restore_with_username_and_password():
+def test_restore_database_dump_runs_mongorestore_with_username_and_password():
     database_config = [
         {
             'name': 'foo',
@@ -251,7 +249,6 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password():
         processes=[extract_process],
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
-        borg_local_path='borg',
     ).once()
 
     module.restore_database_dump(
@@ -270,7 +267,6 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
         processes=[extract_process],
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
-        borg_local_path='borg',
     ).once()
 
     module.restore_database_dump(
@@ -300,7 +296,6 @@ def test_restore_database_dump_without_extract_process_restores_from_disk():
         processes=[],
         output_log_level=logging.DEBUG,
         input_file=None,
-        borg_local_path='borg',
     ).once()
 
     module.restore_database_dump(

tests/unit/hooks/test_mysql.py

@@ -239,7 +239,6 @@ def test_restore_database_dump_runs_mysql_to_restore():
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
         extra_environment=None,
-        borg_local_path='borg',
     ).once()
 
     module.restore_database_dump(
@@ -278,7 +277,6 @@ def test_restore_database_dump_runs_mysql_with_hostname_and_port():
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
         extra_environment=None,
-        borg_local_path='borg',
     ).once()
 
     module.restore_database_dump(
@@ -296,7 +294,6 @@ def test_restore_database_dump_runs_mysql_with_username_and_password():
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
         extra_environment={'MYSQL_PWD': 'trustsome1'},
-        borg_local_path='borg',
     ).once()
 
     module.restore_database_dump(

tests/unit/hooks/test_postgresql.py

@@ -244,7 +244,6 @@ def test_restore_database_dump_runs_pg_restore():
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
         extra_environment={'PGSSLMODE': 'disable'},
-        borg_local_path='borg',
     ).once()
     flexmock(module).should_receive('execute_command').with_args(
         ('psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'),
@@ -296,7 +295,6 @@ def test_restore_database_dump_runs_pg_restore_with_hostname_and_port():
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
         extra_environment={'PGSSLMODE': 'disable'},
-        borg_local_path='borg',
     ).once()
     flexmock(module).should_receive('execute_command').with_args(
         (
@@ -345,7 +343,6 @@ def test_restore_database_dump_runs_pg_restore_with_username_and_password():
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
         extra_environment={'PGPASSWORD': 'trustsome1', 'PGSSLMODE': 'disable'},
-        borg_local_path='borg',
     ).once()
     flexmock(module).should_receive('execute_command').with_args(
         (
@@ -380,7 +377,6 @@ def test_restore_database_dump_runs_psql_for_all_database_dump():
         output_log_level=logging.DEBUG,
         input_file=extract_process.stdout,
         extra_environment={'PGSSLMODE': 'disable'},
-        borg_local_path='borg',
     ).once()
     flexmock(module).should_receive('execute_command').with_args(
         ('psql', '--no-password', '--quiet', '--command', 'ANALYZE'),
@@ -426,7 +422,6 @@ def test_restore_database_dump_without_extract_process_restores_from_disk():
         output_log_level=logging.DEBUG,
         input_file=None,
         extra_environment={'PGSSLMODE': 'disable'},
-        borg_local_path='borg',
     ).once()
     flexmock(module).should_receive('execute_command').with_args(
         ('psql', '--no-password', '--quiet', '--dbname', 'foo', '--command', 'ANALYZE'),

tests/unit/test_execute.py

@@ -289,6 +289,27 @@ def test_execute_command_with_processes_calls_full_command():
     assert output is None
 
 
+def test_execute_command_with_processes_returns_output_with_output_log_level_none():
+    full_command = ['foo', 'bar']
+    processes = (flexmock(),)
+    flexmock(module.os, environ={'a': 'b'})
+    process = flexmock(stdout=None)
+    flexmock(module.subprocess).should_receive('Popen').with_args(
+        full_command,
+        stdin=None,
+        stdout=module.subprocess.PIPE,
+        stderr=module.subprocess.STDOUT,
+        shell=False,
+        env=None,
+        cwd=None,
+    ).and_return(process).once()
+    flexmock(module).should_receive('log_outputs').and_return({process: 'out'})
+
+    output = module.execute_command_with_processes(full_command, processes, output_log_level=None)
+
+    assert output == 'out'
+
+
 def test_execute_command_with_processes_calls_full_command_with_output_file():
     full_command = ['foo', 'bar']
     processes = (flexmock(),)