peter/borgmatic
1
0
Fork 0
mirror of https://github.com/borgmatic-collective/borgmatic.git synced 2026-07-22 02:03:01 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into fix-borg-2-latest-archive

Browse Source
This commit is contained in:
Dan Helfman
2025-09-17 23:00:27 -07:00
parent 97cd3f20df da65063216
commit f3d42d081e
21 changed files with 927 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files
NEWS
+4
View File
@@ -1,5 +1,9 @@
2.0.8.dev0
* #1114: Document systemd configuration changes for the ZFS filesystem hook.
* #1116: Add dumping of database containers via their container names, handy for backing up
database containers from the host. See the documentation for more information:
https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#database-client-on-the-host
* #1116: Add optional database labels to make it easier to find your dumps within a Borg archive.
* #1118: Fix a bug in which Borg hangs during database backup when different filesystems are in
use.
* #1122: To prevent the user from inadvertently excluding the "bootstrap" action's manifest, always
borgmatic/actions/restore.py
+41 -10
View File
@@ -22,8 +22,8 @@ UNSPECIFIED = object()
Dump = collections.namedtuple(
'Dump',
('hook_name', 'data_source_name', 'hostname', 'port'),
defaults=('localhost', None),
('hook_name', 'data_source_name', 'hostname', 'port', 'label', 'container'),
defaults=('localhost', None, None, None),
)
@@ -33,7 +33,13 @@ def dumps_match(first, second, default_port=None):
indicates that the field should match any value. If a default port is given, then consider any
dump having that port to match with a dump having a None port.
'''
for field_name in first._fields:
# label kinda counts as an unique id, if they match ignore host/container/port
if first.label not in {None, UNSPECIFIED} and first.label == second.label:
field_list = ('hook_name', 'data_source_name')
else:
field_list = Dump._fields
for field_name in field_list:
first_value = getattr(first, field_name)
second_value = getattr(second, field_name)
@@ -57,14 +63,17 @@ def render_dump_metadata(dump):
'''
Given a Dump instance, make a display string describing it for use in log messages.
'''
label = dump.label or UNSPECIFIED
name = 'unspecified' if dump.data_source_name is UNSPECIFIED else dump.data_source_name
hostname = dump.hostname or UNSPECIFIED
host = dump.container or dump.hostname or UNSPECIFIED
port = None if dump.port is UNSPECIFIED else dump.port
if port:
metadata = f'{name}@:{port}' if hostname is UNSPECIFIED else f'{name}@{hostname}:{port}'
if label is not UNSPECIFIED:
metadata = f'{name}@{label}'
elif port:
metadata = f'{name}@:{port}' if host is UNSPECIFIED else f'{name}@{host}:{port}'
else:
metadata = f'{name}' if hostname is UNSPECIFIED else f'{name}@{hostname}'
metadata = f'{name}' if host is UNSPECIFIED else f'{name}@{host}'
if dump.hook_name not in {None, UNSPECIFIED}:
return f'{metadata} ({dump.hook_name})'
@@ -101,6 +110,8 @@ def get_configured_data_source(config, restore_dump):
hook_data_source.get('name'),
hook_data_source.get('hostname', 'localhost'),
hook_data_source.get('port'),
hook_data_source.get('label') or UNSPECIFIED,
hook_data_source.get('container'),
),
restore_dump,
default_port,
@@ -172,7 +183,14 @@ def restore_single_dump(
that data source from the archive.
'''
dump_metadata = render_dump_metadata(
Dump(hook_name, data_source['name'], data_source.get('hostname'), data_source.get('port')),
Dump(
hook_name,
data_source['name'],
data_source.get('hostname'),
data_source.get('port'),
data_source.get('label') or UNSPECIFIED,
data_source.get('container'),
),
)
logger.info(f'Restoring data source {dump_metadata}')
@@ -408,6 +426,8 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive):
data_source_name=name,
hostname=restore_arguments.original_hostname or UNSPECIFIED,
port=restore_arguments.original_port,
label=restore_arguments.original_label or UNSPECIFIED,
container=restore_arguments.original_container or UNSPECIFIED,
)
for name in restore_arguments.data_sources or (UNSPECIFIED,)
}
@@ -415,12 +435,16 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive):
or restore_arguments.data_sources
or restore_arguments.original_hostname
or restore_arguments.original_port
or restore_arguments.original_label
or restore_arguments.original_container
else {
Dump(
hook_name=UNSPECIFIED,
data_source_name='all',
hostname=UNSPECIFIED,
port=UNSPECIFIED,
label=UNSPECIFIED,
container=UNSPECIFIED,
),
}
)
@@ -541,6 +565,7 @@ def run_restore(
dumps_actually_restored = set()
connection_params = {
'container': restore_arguments.container,
'hostname': restore_arguments.hostname,
'port': restore_arguments.port,
'username': restore_arguments.username,
@@ -554,13 +579,19 @@ def run_restore(
config,
restore_dump,
)
# For a dump that wasn't found via an exact match in the configuration, try to fallback
# to an "all" data source.
if not found_data_source:
found_data_source = get_configured_data_source(
config,
Dump(restore_dump.hook_name, 'all', restore_dump.hostname, restore_dump.port),
Dump(
restore_dump.hook_name,
'all',
restore_dump.hostname,
restore_dump.port,
restore_dump.label,
restore_dump.container,
),
)
if not found_data_source:
borgmatic/commands/arguments.py
+12
View File
@@ -1475,6 +1475,10 @@ def make_parsers(schema, unparsed_arguments): # noqa: PLR0915
'--port',
help='Database port to restore to. Defaults to the "restore_port" option in borgmatic\'s configuration',
)
restore_group.add_argument(
'--container',
help='Container to restore to. Defaults to the "restore_container" option in borgmatic\'s configuration',
)
restore_group.add_argument(
'--username',
help='Username with which to connect to the database. Defaults to the "restore_username" option in borgmatic\'s configuration',
@@ -1487,10 +1491,18 @@ def make_parsers(schema, unparsed_arguments): # noqa: PLR0915
'--restore-path',
help='Path to restore SQLite database dumps to. Defaults to the "restore_path" option in borgmatic\'s configuration',
)
restore_group.add_argument(
'--original-label',
help='The label where the dump to restore came from, only necessary if you need to disambiguate dumps',
)
restore_group.add_argument(
'--original-hostname',
help='The hostname where the dump to restore came from, only necessary if you need to disambiguate dumps',
)
restore_group.add_argument(
'--original-container',
help='The container where the dump to restore came from, only necessary if you need to disambiguate dumps',
)
restore_group.add_argument(
'--original-port',
type=int,
borgmatic/config/arguments.py
+3 -3
View File
@@ -146,9 +146,9 @@ def prepare_arguments_for_config(global_arguments, schema):
keys = tuple(argument_name.split('.'))
option_type = type_for_option(schema, keys)
# The argument doesn't correspond to any option in the schema, so ignore it. It's
# probably a flag that borgmatic has on the command-line but not in configuration.
if option_type is None:
# The argument doesn't correspond to any option in the schema, or it is a complex argument, so ignore it.
# It's probably a flag that borgmatic has on the command-line but not in configuration.
if option_type in {'object', None}:
continue
prepared_values.append(
borgmatic/config/schema.yaml
+73
View File
@@ -1323,6 +1323,23 @@ properties:
implicitly enables read_special (see above) to support
dump and restore streaming.
example: users
label:
type: string
description: |
Label to identify the database dump in the backup.
example: my_backup_label
container:
type: string
description: |
Container name/id to connect to. When specified the
hostname is ignored. Requires docker/podman CLI.
example: debian_stable
restore_container:
type: string
description: |
Container name/id to restore to. Defaults to the
"container" option.
example: restore_container
hostname:
type: string
description: |
@@ -1524,6 +1541,23 @@ properties:
database hook implicitly enables read_special (see
above) to support dump and restore streaming.
example: users
label:
type: string
description: |
Label to identify the database dump in the backup.
example: my_backup_label
container:
type: string
description: |
Container name/id to connect to. When specified the
hostname is ignored. Requires docker/podman CLI.
example: debian_stable
restore_container:
type: string
description: |
Container name/id to restore to. Defaults to the
"container" option.
example: restore_container
hostname:
type: string
description: |
@@ -1689,6 +1723,23 @@ properties:
database hook implicitly enables read_special (see
above) to support dump and restore streaming.
example: users
label:
type: string
description: |
Label to identify the database dump in the backup.
example: my_backup_label
container:
type: string
description: |
Container name/id to connect to. When specified the
hostname is ignored. Requires docker/podman CLI.
example: debian_stable
restore_container:
type: string
description: |
Container name/id to restore to. Defaults to the
"container" option.
example: restore_container
hostname:
type: string
description: |
@@ -1862,6 +1913,11 @@ properties:
read_special (see above) to support dump and restore
streaming.
example: /var/lib/sqlite/users.db
label:
type: string
description: |
Label to identify the database dump in the backup.
example: my_backup_label
restore_path:
type: string
description: |
@@ -1910,6 +1966,23 @@ properties:
database hook implicitly enables read_special (see
above) to support dump and restore streaming.
example: users
label:
type: string
description: |
Label to identify the database dump in the backup.
example: my_backup_label
container:
type: string
description: |
Container name/id to connect to. When specified the
hostname is ignored. Requires docker/podman CLI.
example: debian_stable
restore_container:
type: string
description: |
Container name/id to restore to. Defaults to the
"container" option.
example: restore_container
hostname:
type: string
description: |
borgmatic/hooks/data_source/config.py
+104
View File
@@ -0,0 +1,104 @@
import json
import logging
import shutil
import subprocess
from borgmatic.execute import execute_command_and_capture_output
IS_A_HOOK = False
logger = logging.getLogger(__name__)
def resolve_database_option(option, data_source, connection_params=None, restore=False):
'''
Resolves a database option from the given data source configuration dict and
connection parameters dict. If restore is set to True it will consider the
`restore_<option>` instead.
Returns the resolved option or None. Can raise a ValueError if the hostname lookup
results in a container IP check.
'''
# Special case `hostname` since it overlaps with `container`
if option == 'hostname':
return get_hostname_from_config(data_source, connection_params, restore)
if connection_params and (value := connection_params.get(option)):
return value
if restore and f'restore_{option}' in data_source:
return data_source[f'restore_{option}']
return data_source.get(option)
def get_hostname_from_config(data_source, connection_params=None, restore=False):
'''
Specialisation of `resolve_database_option` to handle the extra complexity of
the hostname option to also handle containers.
Returns a hostname/IP or raises an ValueError if a container IP lookup fails.
'''
# connection params win, full stop
if connection_params:
if container := connection_params.get('container'):
return get_ip_from_container(container)
if hostname := connection_params.get('hostname'):
return hostname
# ... then try the restore config
if restore:
if 'restore_container' in data_source:
return get_ip_from_container(data_source['restore_container'])
if 'restore_hostname' in data_source:
return data_source['restore_hostname']
# ... and finally fall back to the normal options
if 'container' in data_source:
return get_ip_from_container(data_source['container'])
return data_source.get('hostname')
def get_ip_from_container(container):
'''
Determine the IP for a given container name via podman and docker.
Returns an IP or raises a ValueError if the lookup fails.
'''
engines = (shutil.which(engine) for engine in ('docker', 'podman'))
engines = [engine for engine in engines if engine]
if not engines:
raise ValueError("Neither 'docker' nor 'podman' could be found on the system")
last_error = None
for engine in engines:
try:
output = execute_command_and_capture_output(
(
engine,
'container',
'inspect',
'--format={{json .NetworkSettings}}',
container,
)
)
except subprocess.CalledProcessError as error:
last_error = error
logger.debug(f"Could not find container '{container}' with engine '{engine}'")
continue # Container does not exist
try:
network_data = json.loads(output.strip())
except json.JSONDecodeError as e:
raise ValueError(f'Could not decode JSON output from {engine}') from e
if main_ip := network_data.get('IPAddress'):
return main_ip
# No main IP found, try the networks
for network in network_data.get('Networks', {}).values():
if ip := network.get('IPAddress'):
return ip
if last_error:
raise last_error
raise ValueError(
f"Could not determine ip address for container '{container}'; running in host mode or userspace networking?"
)
borgmatic/hooks/data_source/dump.py
+11 -6
View File
@@ -19,7 +19,9 @@ def make_data_source_dump_path(borgmatic_runtime_directory, data_source_hook_nam
return os.path.join(borgmatic_runtime_directory, data_source_hook_name)
def make_data_source_dump_filename(dump_path, name, hostname=None, port=None):
def make_data_source_dump_filename(
dump_path, name, hostname=None, port=None, container=None, label=None
):
'''
Based on the given dump directory path, data source name, hostname, and port, return a filename
to use for the data source dump. The hostname defaults to localhost.
@@ -29,11 +31,14 @@ def make_data_source_dump_filename(dump_path, name, hostname=None, port=None):
if os.path.sep in name:
raise ValueError(f'Invalid data source name {name}')
return os.path.join(
dump_path,
(hostname or 'localhost') + ('' if port is None else f':{port}'),
name,
)
if container:
hostname = container
elif not hostname:
hostname = 'localhost'
identifier = label if label else hostname + ('' if port is None else f':{port}')
return os.path.join(dump_path, identifier, name)
def write_data_source_dumps_metadata(borgmatic_runtime_directory, hook_name, dumps_metadata):
borgmatic/hooks/data_source/mariadb.py
+27 -21
View File
@@ -12,6 +12,7 @@ from borgmatic.execute import (
execute_command_and_capture_output,
execute_command_with_processes,
)
from borgmatic.hooks.data_source import config as database_config
from borgmatic.hooks.data_source import dump
logger = logging.getLogger(__name__)
@@ -122,6 +123,7 @@ def database_names_to_dump(database, config, username, password, environment, dr
)
extra_options, defaults_extra_filename = parse_extra_options(database.get('list_options'))
password_transport = database.get('password_transport', 'pipe')
hostname = database_config.resolve_database_option('hostname', database)
show_command = (
mariadb_show_command
+ (
@@ -130,9 +132,9 @@ def database_names_to_dump(database, config, username, password, environment, dr
else ()
)
+ extra_options
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--host', hostname) if hostname else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+ (('--protocol', 'tcp') if hostname or 'port' in database else ())
+ (('--user', username) if username and password_transport == 'environment' else ())
+ (('--ssl',) if database.get('tls') is True else ())
+ (('--skip-ssl',) if database.get('tls') is False else ())
@@ -176,8 +178,10 @@ def execute_dump_command(
dump_filename = dump.make_data_source_dump_filename(
dump_path,
database['name'],
database.get('hostname'),
database.get('port'),
hostname=database.get('hostname'),
port=database.get('port'),
container=database.get('container'),
label=database.get('label'),
)
if os.path.exists(dump_filename):
@@ -192,6 +196,7 @@ def execute_dump_command(
)
extra_options, defaults_extra_filename = parse_extra_options(database.get('options'))
password_transport = database.get('password_transport', 'pipe')
hostname = database_config.resolve_database_option('hostname', database)
dump_command = (
mariadb_dump_command
+ (
@@ -201,9 +206,9 @@ def execute_dump_command(
)
+ extra_options
+ (('--add-drop-database',) if database.get('add_drop_database', True) else ())
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--host', hostname) if hostname else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+ (('--protocol', 'tcp') if hostname or 'port' in database else ())
+ (('--user', username) if username and password_transport == 'environment' else ())
+ (('--ssl',) if database.get('tls') is True else ())
+ (('--skip-ssl',) if database.get('tls') is False else ())
@@ -302,6 +307,8 @@ def dump_data_sources(
database_name,
database.get('hostname', 'localhost'),
database.get('port'),
database.get('label'),
database.get('container'),
)
)
renamed_database = copy.copy(database)
@@ -326,6 +333,8 @@ def dump_data_sources(
database['name'],
database.get('hostname', 'localhost'),
database.get('port'),
database.get('label'),
database.get('container'),
)
)
processes.append(
@@ -384,16 +393,16 @@ def make_data_source_dump_patterns(
borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
return (
dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, hostname='*'),
dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, label='*'),
dump.make_data_source_dump_filename(
make_dump_path(borgmatic_runtime_directory),
name,
hostname='*',
label='*',
),
dump.make_data_source_dump_filename(
make_dump_path(borgmatic_source_directory),
name,
hostname='*',
label='*',
),
)
@@ -414,25 +423,22 @@ def restore_data_source_dump(
subprocess.Popen) to produce output to consume.
'''
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
hostname = connection_params['hostname'] or data_source.get(
'restore_hostname',
data_source.get('hostname'),
hostname = database_config.resolve_database_option(
'hostname', data_source, connection_params, restore=True
)
port = str(
connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')),
port = database_config.resolve_database_option(
'port', data_source, connection_params, restore=True
)
tls = data_source.get('restore_tls', data_source.get('tls'))
tls = database_config.resolve_database_option('tls', data_source, restore=True)
username = borgmatic.hooks.credential.parse.resolve_credential(
(
connection_params['username']
or data_source.get('restore_username', data_source.get('username'))
database_config.resolve_database_option(
'username', data_source, connection_params, restore=True
),
config,
)
password = borgmatic.hooks.credential.parse.resolve_credential(
(
connection_params['password']
or data_source.get('restore_password', data_source.get('password'))
database_config.resolve_database_option(
'password', data_source, connection_params, restore=True
),
config,
)
borgmatic/hooks/data_source/mongodb.py
+30 -19
View File
@@ -6,6 +6,7 @@ import borgmatic.borg.pattern
import borgmatic.config.paths
import borgmatic.hooks.credential.parse
from borgmatic.execute import execute_command, execute_command_with_processes
from borgmatic.hooks.data_source import config as database_config
from borgmatic.hooks.data_source import dump
logger = logging.getLogger(__name__)
@@ -63,14 +64,18 @@ def dump_data_sources(
name,
database.get('hostname', 'localhost'),
database.get('port'),
database.get('label'),
database.get('container'),
)
)
dump_filename = dump.make_data_source_dump_filename(
make_dump_path(borgmatic_runtime_directory),
name,
database.get('hostname'),
database.get('port'),
hostname=database.get('hostname'),
port=database.get('port'),
container=database.get('container'),
label=database.get('label'),
)
dump_format = database.get('format', 'archive')
@@ -138,10 +143,11 @@ def build_dump_command(database, config, dump_filename, dump_format):
dump_command = tuple(
shlex.quote(part) for part in shlex.split(database.get('mongodump_command') or 'mongodump')
)
hostname = database_config.resolve_database_option('hostname', database)
return (
dump_command
+ (('--out', shlex.quote(dump_filename)) if dump_format == 'directory' else ())
+ (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ())
+ (('--host', shlex.quote(hostname)) if hostname else ())
+ (('--port', shlex.quote(str(database['port']))) if 'port' in database else ())
+ (
(
@@ -158,7 +164,10 @@ def build_dump_command(database, config, dump_filename, dump_format):
)
+ (('--config', make_password_config_file(password)) if password else ())
+ (
('--authenticationDatabase', shlex.quote(database['authentication_database']))
(
'--authenticationDatabase',
shlex.quote(database['authentication_database']),
)
if 'authentication_database' in database
else ()
)
@@ -200,16 +209,16 @@ def make_data_source_dump_patterns(
borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
return (
dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, hostname='*'),
dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, label='*'),
dump.make_data_source_dump_filename(
make_dump_path(borgmatic_runtime_directory),
name,
hostname='*',
label='*',
),
dump.make_data_source_dump_filename(
make_dump_path(borgmatic_source_directory),
name,
hostname='*',
label='*',
),
)
@@ -237,7 +246,10 @@ def restore_data_source_dump(
dump_filename = dump.make_data_source_dump_filename(
make_dump_path(borgmatic_runtime_directory),
data_source['name'],
data_source.get('hostname'),
hostname=data_source.get('hostname'),
port=data_source.get('port'),
container=data_source.get('container'),
label=data_source.get('label'),
)
restore_command = build_restore_command(
extract_process,
@@ -247,7 +259,7 @@ def restore_data_source_dump(
connection_params,
)
logger.debug(f"Restoring MongoDB database {data_source['name']}{dry_run_label}")
logger.debug(f'Restoring MongoDB database {data_source["name"]}{dry_run_label}')
if dry_run:
return
@@ -265,22 +277,21 @@ def build_restore_command(extract_process, database, config, dump_filename, conn
'''
Return the custom mongorestore_command from a single database configuration.
'''
hostname = connection_params['hostname'] or database.get(
'restore_hostname',
database.get('hostname'),
hostname = database_config.resolve_database_option(
'hostname', database, connection_params, restore=True
)
port = database_config.resolve_database_option(
'port', database, connection_params, restore=True
)
port = str(connection_params['port'] or database.get('restore_port', database.get('port', '')))
username = borgmatic.hooks.credential.parse.resolve_credential(
(
connection_params['username']
or database.get('restore_username', database.get('username'))
database_config.resolve_database_option(
'username', database, connection_params, restore=True
),
config,
)
password = borgmatic.hooks.credential.parse.resolve_credential(
(
connection_params['password']
or database.get('restore_password', database.get('password'))
database_config.resolve_database_option(
'password', database, connection_params, restore=True
),
config,
)
borgmatic/hooks/data_source/mysql.py
+27 -21
View File
@@ -12,6 +12,7 @@ from borgmatic.execute import (
execute_command_and_capture_output,
execute_command_with_processes,
)
from borgmatic.hooks.data_source import config as database_config
from borgmatic.hooks.data_source import dump
logger = logging.getLogger(__name__)
@@ -47,6 +48,7 @@ def database_names_to_dump(database, config, username, password, environment, dr
borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('list_options'))
)
password_transport = database.get('password_transport', 'pipe')
hostname = database_config.resolve_database_option('hostname', database)
show_command = (
mysql_show_command
+ (
@@ -59,9 +61,9 @@ def database_names_to_dump(database, config, username, password, environment, dr
else ()
)
+ extra_options
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--host', hostname) if hostname else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+ (('--protocol', 'tcp') if hostname or 'port' in database else ())
+ (('--user', username) if username and password_transport == 'environment' else ())
+ (('--ssl',) if database.get('tls') is True else ())
+ (('--skip-ssl',) if database.get('tls') is False else ())
@@ -102,8 +104,10 @@ def execute_dump_command(
dump_filename = dump.make_data_source_dump_filename(
dump_path,
database['name'],
database.get('hostname'),
database.get('port'),
hostname=database.get('hostname'),
port=database.get('port'),
container=database.get('container'),
label=database.get('label'),
)
if os.path.exists(dump_filename):
@@ -119,6 +123,7 @@ def execute_dump_command(
borgmatic.hooks.data_source.mariadb.parse_extra_options(database.get('options'))
)
password_transport = database.get('password_transport', 'pipe')
hostname = database_config.resolve_database_option('hostname', database)
dump_command = (
mysql_dump_command
+ (
@@ -132,9 +137,9 @@ def execute_dump_command(
)
+ extra_options
+ (('--add-drop-database',) if database.get('add_drop_database', True) else ())
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--host', hostname) if hostname else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (('--protocol', 'tcp') if 'hostname' in database or 'port' in database else ())
+ (('--protocol', 'tcp') if hostname or 'port' in database else ())
+ (('--user', username) if username and password_transport == 'environment' else ())
+ (('--ssl',) if database.get('tls') is True else ())
+ (('--skip-ssl',) if database.get('tls') is False else ())
@@ -233,6 +238,8 @@ def dump_data_sources(
database_name,
database.get('hostname', 'localhost'),
database.get('port'),
database.get('label'),
database.get('container'),
)
)
renamed_database = copy.copy(database)
@@ -257,6 +264,8 @@ def dump_data_sources(
database['name'],
database.get('hostname', 'localhost'),
database.get('port'),
database.get('label'),
database.get('container'),
)
)
processes.append(
@@ -315,16 +324,16 @@ def make_data_source_dump_patterns(
borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
return (
dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, hostname='*'),
dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, label='*'),
dump.make_data_source_dump_filename(
make_dump_path(borgmatic_runtime_directory),
name,
hostname='*',
label='*',
),
dump.make_data_source_dump_filename(
make_dump_path(borgmatic_source_directory),
name,
hostname='*',
label='*',
),
)
@@ -345,25 +354,22 @@ def restore_data_source_dump(
subprocess.Popen) to produce output to consume.
'''
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
hostname = connection_params['hostname'] or data_source.get(
'restore_hostname',
data_source.get('hostname'),
hostname = database_config.resolve_database_option(
'hostname', data_source, connection_params, restore=True
)
port = str(
connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')),
port = database_config.resolve_database_option(
'port', data_source, connection_params, restore=True
)
tls = data_source.get('restore_tls', data_source.get('tls'))
tls = database_config.resolve_database_option('tls', data_source, restore=True)
username = borgmatic.hooks.credential.parse.resolve_credential(
(
connection_params['username']
or data_source.get('restore_username', data_source.get('username'))
database_config.resolve_database_option(
'username', data_source, connection_params, restore=True
),
config,
)
password = borgmatic.hooks.credential.parse.resolve_credential(
(
connection_params['password']
or data_source.get('restore_password', data_source.get('password'))
database_config.resolve_database_option(
'password', data_source, connection_params, restore=True
),
config,
)
borgmatic/hooks/data_source/postgresql.py
+35 -34
View File
@@ -13,6 +13,7 @@ from borgmatic.execute import (
execute_command_and_capture_output,
execute_command_with_processes,
)
from borgmatic.hooks.data_source import config as database_config
from borgmatic.hooks.data_source import dump
logger = logging.getLogger(__name__)
@@ -32,22 +33,15 @@ def make_environment(database, config, restore_connection_params=None):
'''
environment = dict(os.environ)
try:
if restore_connection_params:
environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential(
(
restore_connection_params.get('password')
or database.get('restore_password', database['password'])
),
config,
)
else:
environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential(
database['password'],
config,
)
except (AttributeError, KeyError):
pass
password = database_config.resolve_database_option(
'password', database, restore_connection_params, restore=restore_connection_params
)
if password:
environment['PGPASSWORD'] = borgmatic.hooks.credential.parse.resolve_credential(
password,
config,
)
if 'ssl_mode' in database:
environment['PGSSLMODE'] = database['ssl_mode']
@@ -91,10 +85,11 @@ def database_names_to_dump(database, config, environment, dry_run):
psql_command = tuple(
shlex.quote(part) for part in shlex.split(database.get('psql_command') or 'psql')
)
hostname = database_config.resolve_database_option('hostname', database)
list_command = (
psql_command
+ ('--list', '--no-password', '--no-psqlrc', '--csv', '--tuples-only')
+ (('--host', database['hostname']) if 'hostname' in database else ())
+ (('--host', hostname) if hostname else ())
+ (('--port', str(database['port'])) if 'port' in database else ())
+ (
(
@@ -172,6 +167,8 @@ def dump_data_sources(
database_name,
database.get('hostname', 'localhost'),
database.get('port'),
database.get('label'),
database.get('container'),
)
)
dump_format = database.get('format', None if database_name == 'all' else 'custom')
@@ -184,8 +181,10 @@ def dump_data_sources(
dump_filename = dump.make_data_source_dump_filename(
dump_path,
database_name,
database.get('hostname'),
database.get('port'),
hostname=database.get('hostname'),
port=database.get('port'),
container=database.get('container'),
label=database.get('label'),
)
if os.path.exists(dump_filename):
@@ -194,6 +193,7 @@ def dump_data_sources(
)
continue
hostname = database_config.resolve_database_option('hostname', database)
command = (
dump_command
+ (
@@ -201,7 +201,7 @@ def dump_data_sources(
'--clean',
'--if-exists',
)
+ (('--host', shlex.quote(database['hostname'])) if 'hostname' in database else ())
+ (('--host', shlex.quote(hostname)) if hostname else ())
+ (('--port', shlex.quote(str(database['port']))) if 'port' in database else ())
+ (
(
@@ -302,16 +302,16 @@ def make_data_source_dump_patterns(
borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
return (
dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, hostname='*'),
dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, label='*'),
dump.make_data_source_dump_filename(
make_dump_path(borgmatic_runtime_directory),
name,
hostname='*',
label='*',
),
dump.make_data_source_dump_filename(
make_dump_path(borgmatic_source_directory),
name,
hostname='*',
label='*',
),
)
@@ -339,17 +339,15 @@ def restore_data_source_dump(
hostname, port, username, and password.
'''
dry_run_label = ' (dry run; not actually restoring anything)' if dry_run else ''
hostname = connection_params['hostname'] or data_source.get(
'restore_hostname',
data_source.get('hostname'),
hostname = database_config.resolve_database_option(
'hostname', data_source, connection_params, restore=True
)
port = str(
connection_params['port'] or data_source.get('restore_port', data_source.get('port', '')),
port = database_config.resolve_database_option(
'port', data_source, connection_params, restore=True
)
username = borgmatic.hooks.credential.parse.resolve_credential(
(
connection_params['username']
or data_source.get('restore_username', data_source.get('username'))
database_config.resolve_database_option(
'username', data_source, connection_params, restore=True
),
config,
)
@@ -358,7 +356,10 @@ def restore_data_source_dump(
dump_filename = dump.make_data_source_dump_filename(
make_dump_path(borgmatic_runtime_directory),
data_source['name'],
data_source.get('hostname'),
hostname=hostname,
port=port,
container=data_source.get('container'),
label=data_source.get('label'),
)
psql_command = tuple(
shlex.quote(part) for part in shlex.split(data_source.get('psql_command') or 'psql')
@@ -367,7 +368,7 @@ def restore_data_source_dump(
psql_command
+ ('--no-password', '--no-psqlrc', '--quiet')
+ (('--host', hostname) if hostname else ())
+ (('--port', port) if port else ())
+ (('--port', str(port)) if port else ())
+ (('--username', username) if username else ())
+ (('--dbname', data_source['name']) if not all_databases else ())
+ (
@@ -388,7 +389,7 @@ def restore_data_source_dump(
+ (('--no-psqlrc',) if use_psql_command else ('--if-exists', '--exit-on-error', '--clean'))
+ (('--dbname', data_source['name']) if not all_databases else ())
+ (('--host', hostname) if hostname else ())
+ (('--port', port) if port else ())
+ (('--port', str(port)) if port else ())
+ (('--username', username) if username else ())
+ (('--no-owner',) if data_source.get('no_owner', False) else ())
+ (
borgmatic/hooks/data_source/sqlite.py
+7 -6
View File
@@ -57,8 +57,7 @@ def dump_data_sources(
database_path = database['path']
dumps_metadata.append(
borgmatic.actions.restore.Dump(
'sqlite_databases',
database['name'],
'sqlite_databases', database['name'], label=database.get('label')
)
)
@@ -71,7 +70,9 @@ def dump_data_sources(
)
dump_path = make_dump_path(borgmatic_runtime_directory)
dump_filename = dump.make_data_source_dump_filename(dump_path, database['name'])
dump_filename = dump.make_data_source_dump_filename(
dump_path, database['name'], label=database.get('label')
)
if os.path.exists(dump_filename):
logger.warning(
@@ -143,16 +144,16 @@ def make_data_source_dump_patterns(
borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
return (
dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, hostname='*'),
dump.make_data_source_dump_filename(make_dump_path('borgmatic'), name, label='*'),
dump.make_data_source_dump_filename(
make_dump_path(borgmatic_runtime_directory),
name,
hostname='*',
label='*',
),
dump.make_data_source_dump_filename(
make_dump_path(borgmatic_source_directory),
name,
hostname='*',
label='*',
),
)
docs/how-to/backup-your-databases.md
+15 -18
View File
@@ -72,8 +72,10 @@ Here's a more involved example that connects to remote databases:
```yaml
postgresql_databases:
- name: users
label: database_server1
hostname: database1.example.org
- name: orders
label: database_server2
hostname: database2.example.org
port: 5433
username: postgres
@@ -212,34 +214,29 @@ these options in the `hooks:` section of your configuration.
#### Database client on the host
But what if borgmatic is running on the host? You can still connect to a
database server container if its ports are properly exposed to the host. For
instance, when running the database container, you can specify `--publish
127.0.0.1:5433:5432` so that it exposes the container's port 5432 to port 5433
on the host (only reachable on localhost, in this case). Or the same thing with
Docker Compose:
But what if borgmatic is running on the host?
```yaml
services:
your-database-server-container-name:
image: postgres
ports:
- 127.0.0.1:5433:5432
```
And then you can configure borgmatic running on the host to connect to the
database:
<span class="minilink minilink-addedin">New in version 2.0.8</span> You can
connect to the database container by specifying its container name or ID:
```yaml
postgresql_databases:
- name: users
hostname: 127.0.0.1
container: your-database-server-container-name
port: 5433
username: postgres
password: trustsome1
```
Alter the ports in these examples to suit your particular database system.
borgmatic uses the `docker`/`podman` CLI to figure out the container IP to
connect to. But `container:` does not work when borgmatic itself is running in a
container; in that case, use `hostname:` as described above.
<span class="minilink minilink-addedin">Prior to version 2.0.8</span> If you're
running an older version of borgmatic on the host, you can publish your database
container ports to the host (e.g. via `docker run --publish` or Compose's
`ports`)—and then configure borgmatic to connect to `localhost` and the
published port.
#### Database client in a running container
tests/end-to-end/commands/docker
Executable
+2
View File
@@ -0,0 +1,2 @@
#!/bin/sh
echo "{\"IPAddress\": \"$4\"}"
tests/end-to-end/hooks/data_source/test_database.py
+127 -2
View File
@@ -94,6 +94,63 @@ sqlite_databases:
return ruamel.yaml.YAML(typ='safe').load(config_yaml)
def write_container_configuration(
source_directory,
config_path,
repository_path,
user_runtime_directory,
):
'''
Write out borgmatic configuration into a file at the config path. Set the options so as to work
for testing. This includes injecting the given repository path, borgmatic source directory for
storing database dumps, and encryption passphrase.
'''
config_yaml = f'''
source_directories:
- {source_directory}
repositories:
- path: {repository_path}
user_runtime_directory: {user_runtime_directory}
encryption_passphrase: "test"
postgresql_databases:
- name: test
container: postgresql
username: postgres
password: test
restore_container: postgresql2
restore_port: 5433
restore_password: test2
mariadb_databases:
- name: test
container: mariadb
username: root
password: test
restore_container: mariadb2
restore_port: 3307
restore_username: root
restore_password: test2
mysql_databases:
- name: test
container: not-actually-mysql
username: root
password: test
mongodb_databases:
- name: test
container: mongodb
username: root
password: test
authentication_database: admin
'''
with open(config_path, 'w') as config_file:
config_file.write(config_yaml)
return ruamel.yaml.YAML(typ='safe').load(config_yaml)
@pytest.mark.parametrize(
'postgresql_all_dump_format,mariadb_mysql_all_dump_format',
(
@@ -116,6 +173,7 @@ def write_custom_restore_configuration(
for testing with custom restore options. This includes a custom restore_hostname, restore_port,
restore_username, restore_password and restore_path.
'''
config_yaml = f'''
source_directories:
- {source_directory}
@@ -212,8 +270,13 @@ postgresql_databases:
def get_connection_params(database, use_restore_options=False):
hostname = (database.get('restore_hostname') if use_restore_options else None) or database.get(
'hostname',
hostname = (
database.get('restore_container', database.get('restore_hostname'))
if use_restore_options
else None
) or database.get(
'container',
database.get('hostname'),
)
port = (database.get('restore_port') if use_restore_options else None) or database.get('port')
username = (database.get('restore_username') if use_restore_options else None) or database.get(
@@ -656,3 +719,65 @@ def test_database_dump_with_error_causes_borgmatic_to_exit():
finally:
os.chdir(original_working_directory)
shutil.rmtree(temporary_directory)
def test_database_dump_and_restore_containers():
# Create a Borg repository.
temporary_directory = tempfile.mkdtemp()
repository_path = os.path.join(temporary_directory, 'test.borg')
original_working_directory = os.getcwd()
original_path = os.environ.get('PATH', '')
os.environ['PATH'] = f'/app/tests/end-to-end/commands:{original_path}'
try:
config_path = os.path.join(temporary_directory, 'test.yaml')
config = write_container_configuration(
temporary_directory,
config_path,
repository_path,
temporary_directory,
)
create_test_tables(config)
select_test_tables(config)
subprocess.check_call(
[
'borgmatic',
'-v',
'2',
'--config',
config_path,
'repo-create',
'--encryption',
'repokey',
],
)
# Run borgmatic to generate a backup archive including database dumps.
subprocess.check_call(['borgmatic', 'create', '--config', config_path, '-v', '2'])
# Get the created archive name.
output = subprocess.check_output(
['borgmatic', '--config', config_path, 'list', '--json'],
).decode(sys.stdout.encoding)
parsed_output = json.loads(output)
assert len(parsed_output) == 1
assert len(parsed_output[0]['archives']) == 1
archive_name = parsed_output[0]['archives'][0]['archive']
# Restore the databases from the archive.
drop_test_tables(config)
subprocess.check_call(
['borgmatic', '-v', '2', '--config', config_path, 'restore', '--archive', archive_name],
)
# Ensure the test tables have actually been restored.
select_test_tables(config, use_restore_options=True)
finally:
os.chdir(original_working_directory)
shutil.rmtree(temporary_directory)
drop_test_tables(config)
os.environ['PATH'] = original_path
tests/integration/hooks/data_source/__init__.py
View File
tests/integration/hooks/data_source/test_config.py
+54
View File
@@ -0,0 +1,54 @@
from flexmock import flexmock
from borgmatic.hooks.data_source import config as module
def test_resolve_database_option_gets_hostname_from_container_ip():
data_source = {
'container': 'original_container',
'hostname': 'original_hostname',
'restore_container': 'restore_container',
'restore_hostname': 'restore_hostname',
}
flexmock(module).should_receive('get_ip_from_container').with_args(
'original_container'
).and_return('container_ip_1')
assert module.resolve_database_option('hostname', data_source) == 'container_ip_1'
def test_resolve_database_option_gets_hostname_from_connection_params_container_ip():
data_source = {
'container': 'original_container',
'hostname': 'original_hostname',
'restore_container': 'restore_container',
'restore_hostname': 'restore_hostname',
}
connection_params = {'container': 'connection_container', 'hostname': 'connection_hostname'}
flexmock(module).should_receive('get_ip_from_container').with_args('original_container').never()
flexmock(module).should_receive('get_ip_from_container').with_args(
'connection_container'
).and_return('container_ip_2')
assert (
module.resolve_database_option('hostname', data_source, connection_params)
== 'container_ip_2'
)
def test_resolve_database_option_gets_hostname_from_restore_container_ip():
data_source = {
'container': 'original_container',
'hostname': 'original_hostname',
'restore_container': 'restore_container',
'restore_hostname': 'restore_hostname',
}
flexmock(module).should_receive('get_ip_from_container').with_args('original_container').never()
flexmock(module).should_receive('get_ip_from_container').with_args(
'restore_container'
).and_return('container_ip_3')
assert module.resolve_database_option('hostname', data_source, restore=True) == 'container_ip_3'
tests/unit/actions/test_restore.py
+166 -21
View File
@@ -126,6 +126,24 @@ import borgmatic.actions.restore as module
5432,
False,
),
(
module.Dump('postgresql_databases', 'foo', 'some_host1', 5433, 'unique'),
module.Dump('postgresql_databases', 'foo', 'some_host2', None, 'unique'),
5432,
True,
),
(
module.Dump('postgresql_databases', 'foo', 'some_host1', 5433, 'unique'),
module.Dump(module.UNSPECIFIED, 'foo', 'some_host2', None, 'unique'),
5432,
True,
),
(
module.Dump('postgresql_databases', 'foo', container='container'),
module.Dump('postgresql_databases', 'foo', container='container'),
5432,
True,
),
),
)
def test_dumps_match_compares_two_dumps_while_respecting_unspecified_values(
@@ -172,6 +190,14 @@ def test_dumps_match_compares_two_dumps_while_respecting_unspecified_values(
module.Dump('postgresql_databases', 'foo', 'host', module.UNSPECIFIED),
'foo@host (postgresql_databases)',
),
(
module.Dump('postgresql_databases', 'foo', 'host', 1234, 'label'),
'foo@label (postgresql_databases)',
),
(
module.Dump('postgresql_databases', 'foo', container='container'),
'foo@container (postgresql_databases)',
),
(
module.Dump(
module.UNSPECIFIED,
@@ -192,7 +218,7 @@ def test_get_configured_data_source_matches_data_source_with_restore_dump():
flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').and_return(default_port)
flexmock(module).should_receive('dumps_match').and_return(False)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump('postgresql_databases', 'bar'),
module.Dump('postgresql_databases', 'bar', label=module.UNSPECIFIED),
module.Dump('postgresql_databases', 'bar'),
default_port=default_port,
).and_return(True)
@@ -239,7 +265,7 @@ def test_get_configured_data_source_with_multiple_matching_data_sources_errors()
flexmock(module.borgmatic.hooks.dispatch).should_receive('call_hook').and_return(default_port)
flexmock(module).should_receive('dumps_match').and_return(False)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump('postgresql_databases', 'bar'),
module.Dump('postgresql_databases', 'bar', label=module.UNSPECIFIED),
module.Dump('postgresql_databases', 'bar'),
default_port=default_port,
).and_return(True)
@@ -720,11 +746,23 @@ def test_get_dumps_to_restore_gets_requested_dumps_found_in_archive():
}
flexmock(module).should_receive('dumps_match').and_return(False)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump(module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED),
module.Dump(
module.UNSPECIFIED,
'foo',
hostname=module.UNSPECIFIED,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'foo'),
).and_return(True)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump(module.UNSPECIFIED, 'bar', hostname=module.UNSPECIFIED),
module.Dump(
module.UNSPECIFIED,
'bar',
hostname=module.UNSPECIFIED,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'bar'),
).and_return(True)
@@ -734,6 +772,8 @@ def test_get_dumps_to_restore_gets_requested_dumps_found_in_archive():
data_sources=['foo', 'bar'],
original_hostname=None,
original_port=None,
original_label=None,
original_container=None,
),
dumps_from_archive=dumps_from_archive,
) == {
@@ -747,18 +787,31 @@ def test_get_dumps_to_restore_raises_for_requested_dumps_missing_from_archive():
module.Dump('postgresql_databases', 'foo'),
}
flexmock(module).should_receive('dumps_match').and_return(False)
flexmock(module).should_receive('render_dump_metadata').and_return('test')
flexmock(module).should_receive('dumps_match').with_args(
module.Dump(
module.UNSPECIFIED,
'foo',
hostname=module.UNSPECIFIED,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'foo'),
).and_return(True)
flexmock(module).should_receive('render_dump_metadata').and_return('test').once()
with pytest.raises(ValueError):
with pytest.raises(ValueError) as exc_info:
module.get_dumps_to_restore(
restore_arguments=flexmock(
hook=None,
data_sources=['foo', 'bar'],
original_hostname=None,
original_port=None,
original_label=None,
original_container=None,
),
dumps_from_archive=dumps_from_archive,
)
assert 'dump test missing from archive' in str(exc_info.value)
def test_get_dumps_to_restore_without_requested_dumps_finds_all_archive_dumps():
@@ -775,6 +828,8 @@ def test_get_dumps_to_restore_without_requested_dumps_finds_all_archive_dumps():
data_sources=[],
original_hostname=None,
original_port=None,
original_label=None,
original_container=None,
),
dumps_from_archive=dumps_from_archive,
)
@@ -789,11 +844,23 @@ def test_get_dumps_to_restore_with_all_in_requested_dumps_finds_all_archive_dump
}
flexmock(module).should_receive('dumps_match').and_return(False)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump(module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED),
module.Dump(
module.UNSPECIFIED,
'foo',
hostname=module.UNSPECIFIED,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'foo'),
).and_return(True)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump(module.UNSPECIFIED, 'bar', hostname=module.UNSPECIFIED),
module.Dump(
module.UNSPECIFIED,
'bar',
hostname=module.UNSPECIFIED,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'bar'),
).and_return(True)
@@ -804,6 +871,8 @@ def test_get_dumps_to_restore_with_all_in_requested_dumps_finds_all_archive_dump
data_sources=['all'],
original_hostname=None,
original_port=None,
original_label=None,
original_container=None,
),
dumps_from_archive=dumps_from_archive,
)
@@ -818,11 +887,23 @@ def test_get_dumps_to_restore_with_all_in_requested_dumps_plus_additional_reques
}
flexmock(module).should_receive('dumps_match').and_return(False)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump(module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED),
module.Dump(
module.UNSPECIFIED,
'foo',
hostname=module.UNSPECIFIED,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'foo'),
).and_return(True)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump(module.UNSPECIFIED, 'bar', hostname=module.UNSPECIFIED),
module.Dump(
module.UNSPECIFIED,
'bar',
hostname=module.UNSPECIFIED,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'bar'),
).and_return(True)
@@ -833,6 +914,8 @@ def test_get_dumps_to_restore_with_all_in_requested_dumps_plus_additional_reques
data_sources=['all', 'foo', 'bar'],
original_hostname=None,
original_port=None,
original_label=None,
original_container=None,
),
dumps_from_archive=dumps_from_archive,
)
@@ -843,48 +926,72 @@ def test_get_dumps_to_restore_with_all_in_requested_dumps_plus_additional_reques
def test_get_dumps_to_restore_raises_for_multiple_matching_dumps_in_archive():
flexmock(module).should_receive('dumps_match').and_return(False)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump(module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED),
module.Dump(
module.UNSPECIFIED,
'foo',
hostname=module.UNSPECIFIED,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'foo'),
).and_return(True)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump(module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED),
module.Dump(
module.UNSPECIFIED,
'foo',
hostname=module.UNSPECIFIED,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('mariadb_databases', 'foo'),
).and_return(True)
flexmock(module).should_receive('render_dump_metadata').and_return('test')
with pytest.raises(ValueError):
with pytest.raises(ValueError) as exc_info:
module.get_dumps_to_restore(
restore_arguments=flexmock(
hook=None,
data_sources=['foo'],
original_hostname=None,
original_port=None,
original_label=None,
original_container=None,
),
dumps_from_archive={
module.Dump('postgresql_databases', 'foo'),
module.Dump('mariadb_databases', 'foo'),
},
)
assert 'Try adding flags to disambiguate.' in str(exc_info.value)
def test_get_dumps_to_restore_raises_for_all_in_requested_dumps_and_requested_dumps_missing_from_archive():
flexmock(module).should_receive('dumps_match').and_return(False)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump(module.UNSPECIFIED, 'foo', hostname=module.UNSPECIFIED),
module.Dump(
module.UNSPECIFIED,
'foo',
hostname=module.UNSPECIFIED,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'foo'),
).and_return(True)
flexmock(module).should_receive('render_dump_metadata').and_return('test')
flexmock(module).should_receive('render_dump_metadata').and_return('test').once()
with pytest.raises(ValueError):
with pytest.raises(ValueError) as exc_info:
module.get_dumps_to_restore(
restore_arguments=flexmock(
hook=None,
data_sources=['all', 'foo', 'bar'],
original_hostname=None,
original_port=None,
original_label=None,
original_container=None,
),
dumps_from_archive={module.Dump('postresql_databases', 'foo')},
dumps_from_archive={module.Dump('postgresql_databases', 'foo')},
)
assert 'dump test missing from archive' in str(exc_info.value)
def test_get_dumps_to_restore_with_requested_hook_name_filters_dumps_found_in_archive():
@@ -895,7 +1002,13 @@ def test_get_dumps_to_restore_with_requested_hook_name_filters_dumps_found_in_ar
}
flexmock(module).should_receive('dumps_match').and_return(False)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump('postgresql_databases', 'foo', hostname=module.UNSPECIFIED),
module.Dump(
'postgresql_databases',
'foo',
hostname=module.UNSPECIFIED,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'foo'),
).and_return(True)
@@ -905,6 +1018,8 @@ def test_get_dumps_to_restore_with_requested_hook_name_filters_dumps_found_in_ar
data_sources=['foo'],
original_hostname=None,
original_port=None,
original_label=None,
original_container=None,
),
dumps_from_archive=dumps_from_archive,
) == {
@@ -920,7 +1035,13 @@ def test_get_dumps_to_restore_with_requested_shortened_hook_name_filters_dumps_f
}
flexmock(module).should_receive('dumps_match').and_return(False)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump('postgresql_databases', 'foo', hostname=module.UNSPECIFIED),
module.Dump(
'postgresql_databases',
'foo',
hostname=module.UNSPECIFIED,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'foo'),
).and_return(True)
@@ -930,6 +1051,8 @@ def test_get_dumps_to_restore_with_requested_shortened_hook_name_filters_dumps_f
data_sources=['foo'],
original_hostname=None,
original_port=None,
original_label=None,
original_container=None,
),
dumps_from_archive=dumps_from_archive,
) == {
@@ -945,7 +1068,13 @@ def test_get_dumps_to_restore_with_requested_hostname_filters_dumps_found_in_arc
}
flexmock(module).should_receive('dumps_match').and_return(False)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump('postgresql_databases', 'foo', 'host'),
module.Dump(
'postgresql_databases',
'foo',
'host',
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'foo', 'host'),
).and_return(True)
@@ -955,6 +1084,8 @@ def test_get_dumps_to_restore_with_requested_hostname_filters_dumps_found_in_arc
data_sources=['foo'],
original_hostname='host',
original_port=None,
original_label=None,
original_container=None,
),
dumps_from_archive=dumps_from_archive,
) == {
@@ -970,7 +1101,14 @@ def test_get_dumps_to_restore_with_requested_port_filters_dumps_found_in_archive
}
flexmock(module).should_receive('dumps_match').and_return(False)
flexmock(module).should_receive('dumps_match').with_args(
module.Dump('postgresql_databases', 'foo', 'host', 1234),
module.Dump(
'postgresql_databases',
'foo',
'host',
1234,
label=module.UNSPECIFIED,
container=module.UNSPECIFIED,
),
module.Dump('postgresql_databases', 'foo', 'host', 1234),
).and_return(True)
@@ -980,6 +1118,8 @@ def test_get_dumps_to_restore_with_requested_port_filters_dumps_found_in_archive
data_sources=['foo'],
original_hostname='host',
original_port=1234,
original_label=None,
original_container=None,
),
dumps_from_archive=dumps_from_archive,
) == {
@@ -1087,6 +1227,7 @@ def test_run_restore_restores_each_data_source():
username=None,
password=None,
restore_path=None,
container=None,
),
global_arguments=flexmock(dry_run=False),
local_path=flexmock(),
@@ -1171,6 +1312,7 @@ def test_run_restore_restores_data_source_by_falling_back_to_all_name():
username=None,
password=None,
restore_path=None,
container=None,
),
global_arguments=flexmock(dry_run=False),
local_path=flexmock(),
@@ -1252,6 +1394,7 @@ def test_run_restore_restores_data_source_configured_with_all_name():
username=None,
password=None,
restore_path=None,
container=None,
),
global_arguments=flexmock(dry_run=False),
local_path=flexmock(),
@@ -1333,6 +1476,7 @@ def test_run_restore_skips_missing_data_source():
username=None,
password=None,
restore_path=None,
container=None,
),
global_arguments=flexmock(dry_run=False),
local_path=flexmock(),
@@ -1410,6 +1554,7 @@ def test_run_restore_restores_data_sources_from_different_hooks():
username=None,
password=None,
restore_path=None,
container=None,
),
global_arguments=flexmock(dry_run=False),
local_path=flexmock(),
tests/unit/config/test_arguments.py
+13
View File
@@ -209,6 +209,19 @@ def test_prepare_arguments_for_config_skips_option_with_none_value():
) == ((('other_option',), 'value2'),)
def test_prepare_arguments_for_config_skips_option_with_complex_schema():
assert module.prepare_arguments_for_config(
global_arguments=flexmock(my_option='value1', other_option='value2'),
schema={
'type': 'object',
'properties': {
'my_option': {'type': 'object', 'properties': {'sub_option': {'type': 'string'}}},
'other_option': {'type': 'string'},
},
},
) == ((('other_option',), 'value2'),)
def test_prepare_arguments_for_config_skips_option_missing_from_schema():
assert module.prepare_arguments_for_config(
global_arguments=flexmock(**{'my_option.sub_option': 'value1', 'other_option': 'value2'}),
tests/unit/hooks/data_source/test_config.py
+157
View File
@@ -0,0 +1,157 @@
from subprocess import CalledProcessError
import pytest
from flexmock import flexmock
from borgmatic.hooks.data_source import config as module
def test_resolve_database_option_uses_config_value():
data_source = {'option': 'original_value', 'restore_option': 'restore_value'}
assert module.resolve_database_option('option', data_source) == 'original_value'
def test_resolve_database_option_with_restore_uses_restore_value():
data_source = {'option': 'original_value', 'restore_option': 'restore_value'}
assert module.resolve_database_option('option', data_source, restore=True) == 'restore_value'
def test_resolve_database_option_with_connection_params_uses_connection_params_value():
data_source = {'option': 'original_value', 'restore_option': 'restore_value'}
connection_params = {'option': 'connection_value'}
assert (
module.resolve_database_option('option', data_source, connection_params)
== 'connection_value'
)
def test_resolve_database_option_with_restore_and_connection_params_uses_connection_params_value():
data_source = {'option': 'original_value', 'restore_option': 'restore_value'}
connection_params = {'option': 'connection_value'}
assert (
module.resolve_database_option('option', data_source, connection_params, restore=True)
== 'connection_value'
)
def test_resolve_database_option_with_hostname_uses_hostname_specific_function():
data_source = {'hostname': 'original_value'}
connection_params = {'hostname': 'connection_value'}
flexmock(module).should_receive('get_hostname_from_config').and_return('special_value').once()
assert (
module.resolve_database_option('hostname', data_source, connection_params, restore=True)
== 'special_value'
)
def test_get_hostname_from_config_gets_container_ip():
data_source = {
'container': 'original_container',
'hostname': 'original_hostname',
'restore_container': 'restore_container',
'restore_hostname': 'restore_hostname',
}
flexmock(module).should_receive('get_ip_from_container').with_args(
'original_container'
).and_return('container_ip_1')
assert module.get_hostname_from_config(data_source) == 'container_ip_1'
def test_get_hostname_from_config_gets_connection_params_container_ip():
data_source = {
'container': 'original_container',
'hostname': 'original_hostname',
'restore_container': 'restore_container',
'restore_hostname': 'restore_hostname',
}
connection_params = {'container': 'connection_container', 'hostname': 'connection_hostname'}
flexmock(module).should_receive('get_ip_from_container').with_args('original_container').never()
flexmock(module).should_receive('get_ip_from_container').with_args(
'connection_container'
).and_return('container_ip_2')
assert module.get_hostname_from_config(data_source, connection_params) == 'container_ip_2'
def test_get_hostname_from_config_gets_restore_container_ip():
data_source = {
'container': 'original_container',
'hostname': 'original_hostname',
'restore_container': 'restore_container',
'restore_hostname': 'restore_hostname',
}
flexmock(module).should_receive('get_ip_from_container').with_args('original_container').never()
flexmock(module).should_receive('get_ip_from_container').with_args(
'restore_container'
).and_return('container_ip_3')
assert module.get_hostname_from_config(data_source, restore=True) == 'container_ip_3'
def test_get_ip_from_container_without_engines_errors():
flexmock(module.shutil).should_receive('which').and_return(None).and_return(None)
with pytest.raises(ValueError):
module.get_ip_from_container('yolo')
def test_get_ip_from_container_parses_top_level_ip_address():
flexmock(module.shutil).should_receive('which').and_return(None).and_return('/usr/bin/podman')
flexmock(module).should_receive('execute_command_and_capture_output').and_return(
'{"IPAddress": "1.2.3.4"}'
)
assert module.get_ip_from_container('yolo') == '1.2.3.4'
def test_get_ip_from_container_parses_network_ip_address():
flexmock(module.shutil).should_receive('which').and_return(None).and_return('/usr/bin/podman')
flexmock(module).should_receive('execute_command_and_capture_output').and_return(
'{"Networks": {"my_network": {"IPAddress": "5.6.7.8"}}}'
)
assert module.get_ip_from_container('yolo') == '5.6.7.8'
def test_get_ip_from_container_without_container_errors():
flexmock(module.shutil).should_receive('which').and_return('/usr/bin/podman')
flexmock(module).should_receive('execute_command_and_capture_output').and_raise(
CalledProcessError, 1, ['/usr/bin/podman', 'inspect', 'yolo'], None, 'No such object'
)
with pytest.raises(CalledProcessError):
module.get_ip_from_container('does not exist')
def test_get_ip_from_container_without_network_errors():
flexmock(module.shutil).should_receive('which').and_return(None).and_return('/usr/bin/podman')
flexmock(module).should_receive('execute_command_and_capture_output').and_return('{}')
with pytest.raises(ValueError) as exc_info:
module.get_ip_from_container('yolo')
assert 'Could not determine ip address for container' in str(exc_info.value)
def test_get_ip_from_container_with_broken_output_errors():
flexmock(module.shutil).should_receive('which').and_return(None).and_return('/usr/bin/podman')
flexmock(module).should_receive('execute_command_and_capture_output').and_return('abc')
with pytest.raises(ValueError) as exc_info:
module.get_ip_from_container('yolo')
assert 'Could not decode JSON output' in str(exc_info.value)
tests/unit/hooks/data_source/test_dump.py
+19 -1
View File
@@ -25,6 +25,24 @@ def test_make_data_source_dump_filename_uses_name_and_hostname_and_port():
)
def test_make_data_source_dump_filename_uses_label():
assert (
module.make_data_source_dump_filename(
'databases', 'test', 'hostname', 1234, label='custom_label'
)
== 'databases/custom_label/test'
)
def test_make_data_source_dump_filename_uses_container():
assert (
module.make_data_source_dump_filename(
'databases', 'test', 'hostname', 1234, container='container'
)
== 'databases/container:1234/test'
)
def test_make_data_source_dump_filename_without_hostname_defaults_to_localhost():
assert module.make_data_source_dump_filename('databases', 'test') == 'databases/localhost/test'
@@ -51,7 +69,7 @@ def test_write_data_source_dumps_metadata_writes_json_to_file():
assert (
dumps_stream.getvalue()
== '{"dumps": [{"data_source_name": "foo", "hook_name": "databases", "hostname": "localhost", "port": null}, {"data_source_name": "bar", "hook_name": "databases", "hostname": "localhost", "port": null}]}'
== '{"dumps": [{"container": null, "data_source_name": "foo", "hook_name": "databases", "hostname": "localhost", "label": null, "port": null}, {"container": null, "data_source_name": "bar", "hook_name": "databases", "hostname": "localhost", "label": null, "port": null}]}'
)