mirror of
https://github.com/borgmatic-collective/borgmatic.git
synced 2026-07-28 04:13:02 +02:00
Compare commits
9
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
55520227dd | ||
|
|
468b2537f1 | ||
|
|
1192451718 | ||
|
|
06e24dfbc1 | ||
|
|
4f0162d5f2 | ||
|
|
ed58bc2943 | ||
|
|
cab6c3ee9b | ||
|
|
65c17c91b3 | ||
|
|
546c0bf83b |
+6
-1
@@ -1,2 +1,7 @@
|
|||||||
|
# This file only applies to the source dist tarball, not the built wheel.
|
||||||
include borgmatic/config/schema.yaml
|
include borgmatic/config/schema.yaml
|
||||||
graft sample/systemd
|
graft docs
|
||||||
|
graft sample
|
||||||
|
graft scripts
|
||||||
|
graft tests
|
||||||
|
global-exclude *.py[co]
|
||||||
|
|||||||
@@ -1,3 +1,13 @@
|
|||||||
|
2.0.9.dev0
|
||||||
|
* #1123: Add loading of systemd credentials even when running borgmatic outside of a systemd
|
||||||
|
service.
|
||||||
|
* #1134: Add a "steps" option to run command hooks around particular sub-action steps like
|
||||||
|
individual checks.
|
||||||
|
* #1149: Add support for Python 3.14.
|
||||||
|
* #1149: Include automated tests in the source dist tarball uploaded to PyPI.
|
||||||
|
* #1151: Fix snapshotting in the ZFS, Btrfs, and LVM hooks to play nicely with the Borg 1.4+
|
||||||
|
"slashdot" hack within source directory paths.
|
||||||
|
|
||||||
2.0.8
|
2.0.8
|
||||||
* #1114: Document systemd configuration changes for the ZFS filesystem hook.
|
* #1114: Document systemd configuration changes for the ZFS filesystem hook.
|
||||||
* #1116: Add dumping of database containers via their container names, handy for backing up
|
* #1116: Add dumping of database containers via their container names, handy for backing up
|
||||||
|
|||||||
+62
-30
@@ -744,6 +744,7 @@ def run_check(
|
|||||||
global_arguments,
|
global_arguments,
|
||||||
local_path,
|
local_path,
|
||||||
remote_path,
|
remote_path,
|
||||||
|
hook_context,
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Run the "check" action for the given repository.
|
Run the "check" action for the given repository.
|
||||||
@@ -783,44 +784,75 @@ def run_check(
|
|||||||
archives_check_id,
|
archives_check_id,
|
||||||
)
|
)
|
||||||
borg_specific_checks = set(checks).intersection({'repository', 'archives', 'data'})
|
borg_specific_checks = set(checks).intersection({'repository', 'archives', 'data'})
|
||||||
|
working_directory = borgmatic.config.paths.get_working_directory(config)
|
||||||
|
|
||||||
if borg_specific_checks:
|
if borg_specific_checks:
|
||||||
borgmatic.borg.check.check_archives(
|
with borgmatic.hooks.command.Before_after_hooks(
|
||||||
repository['path'],
|
command_hooks=config.get('commands'),
|
||||||
config,
|
before_after='step',
|
||||||
local_borg_version,
|
umask=config.get('umask'),
|
||||||
check_arguments,
|
working_directory=working_directory,
|
||||||
global_arguments,
|
dry_run=global_arguments.dry_run,
|
||||||
borg_specific_checks,
|
action_names=('check',),
|
||||||
archive_filter_flags,
|
step_names=('archives_repository_data',),
|
||||||
local_path=local_path,
|
**hook_context,
|
||||||
remote_path=remote_path,
|
):
|
||||||
)
|
borgmatic.borg.check.check_archives(
|
||||||
for check in borg_specific_checks:
|
repository['path'],
|
||||||
write_check_time(make_check_time_path(config, repository_id, check, archives_check_id))
|
config,
|
||||||
|
local_borg_version,
|
||||||
|
check_arguments,
|
||||||
|
global_arguments,
|
||||||
|
borg_specific_checks,
|
||||||
|
archive_filter_flags,
|
||||||
|
local_path=local_path,
|
||||||
|
remote_path=remote_path,
|
||||||
|
)
|
||||||
|
for check in borg_specific_checks:
|
||||||
|
write_check_time(make_check_time_path(config, repository_id, check, archives_check_id))
|
||||||
|
|
||||||
if 'extract' in checks:
|
if 'extract' in checks:
|
||||||
borgmatic.borg.extract.extract_last_archive_dry_run(
|
with borgmatic.hooks.command.Before_after_hooks(
|
||||||
config,
|
command_hooks=config.get('commands'),
|
||||||
local_borg_version,
|
before_after='step',
|
||||||
global_arguments,
|
umask=config.get('umask'),
|
||||||
repository['path'],
|
working_directory=working_directory,
|
||||||
config.get('lock_wait'),
|
dry_run=global_arguments.dry_run,
|
||||||
local_path,
|
action_names=('check',),
|
||||||
remote_path,
|
step_names=('extract',),
|
||||||
)
|
**hook_context,
|
||||||
write_check_time(make_check_time_path(config, repository_id, 'extract'))
|
):
|
||||||
|
borgmatic.borg.extract.extract_last_archive_dry_run(
|
||||||
if 'spot' in checks:
|
|
||||||
with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
|
|
||||||
spot_check(
|
|
||||||
repository,
|
|
||||||
config,
|
config,
|
||||||
local_borg_version,
|
local_borg_version,
|
||||||
global_arguments,
|
global_arguments,
|
||||||
|
repository['path'],
|
||||||
|
config.get('lock_wait'),
|
||||||
local_path,
|
local_path,
|
||||||
remote_path,
|
remote_path,
|
||||||
borgmatic_runtime_directory,
|
|
||||||
)
|
)
|
||||||
|
write_check_time(make_check_time_path(config, repository_id, 'extract'))
|
||||||
|
|
||||||
write_check_time(make_check_time_path(config, repository_id, 'spot'))
|
if 'spot' in checks:
|
||||||
|
with borgmatic.hooks.command.Before_after_hooks(
|
||||||
|
command_hooks=config.get('commands'),
|
||||||
|
before_after='step',
|
||||||
|
umask=config.get('umask'),
|
||||||
|
working_directory=working_directory,
|
||||||
|
dry_run=global_arguments.dry_run,
|
||||||
|
action_names=('check',),
|
||||||
|
step_names=('spot',),
|
||||||
|
**hook_context,
|
||||||
|
):
|
||||||
|
with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
|
||||||
|
spot_check(
|
||||||
|
repository,
|
||||||
|
config,
|
||||||
|
local_borg_version,
|
||||||
|
global_arguments,
|
||||||
|
local_path,
|
||||||
|
remote_path,
|
||||||
|
borgmatic_runtime_directory,
|
||||||
|
)
|
||||||
|
|
||||||
|
write_check_time(make_check_time_path(config, repository_id, 'spot'))
|
||||||
|
|||||||
@@ -463,6 +463,7 @@ def run_actions( # noqa: PLR0912, PLR0915
|
|||||||
global_arguments,
|
global_arguments,
|
||||||
local_path,
|
local_path,
|
||||||
remote_path,
|
remote_path,
|
||||||
|
hook_context,
|
||||||
)
|
)
|
||||||
elif action_name == 'extract':
|
elif action_name == 'extract':
|
||||||
borgmatic.actions.extract.run_extract(
|
borgmatic.actions.extract.run_extract(
|
||||||
|
|||||||
@@ -1140,6 +1140,7 @@ properties:
|
|||||||
before:
|
before:
|
||||||
type: string
|
type: string
|
||||||
enum:
|
enum:
|
||||||
|
- step
|
||||||
- action
|
- action
|
||||||
- repository
|
- repository
|
||||||
- configuration
|
- configuration
|
||||||
@@ -1148,6 +1149,8 @@ properties:
|
|||||||
Name for the point in borgmatic's execution that
|
Name for the point in borgmatic's execution that
|
||||||
the commands should be run before (required if
|
the commands should be run before (required if
|
||||||
"after" isn't set):
|
"after" isn't set):
|
||||||
|
* "step" runs before a sub-action step for each
|
||||||
|
repository, e.g. for an individual check.
|
||||||
* "action" runs before each action for each
|
* "action" runs before each action for each
|
||||||
repository.
|
repository.
|
||||||
* "repository" runs before all actions for each
|
* "repository" runs before all actions for each
|
||||||
@@ -1188,6 +1191,18 @@ properties:
|
|||||||
List of actions for which the commands will be
|
List of actions for which the commands will be
|
||||||
run. Defaults to running for all actions.
|
run. Defaults to running for all actions.
|
||||||
example: [create, prune, compact, check]
|
example: [create, prune, compact, check]
|
||||||
|
steps:
|
||||||
|
type: array
|
||||||
|
items:
|
||||||
|
type: string
|
||||||
|
enum:
|
||||||
|
- archives_repository_data
|
||||||
|
- extract
|
||||||
|
- spot
|
||||||
|
description: |
|
||||||
|
List of sub-action steps for which the commands
|
||||||
|
will be run. Defaults to running for all steps.
|
||||||
|
example: [extract, spot]
|
||||||
run:
|
run:
|
||||||
type: array
|
type: array
|
||||||
items:
|
items:
|
||||||
@@ -1203,6 +1218,7 @@ properties:
|
|||||||
after:
|
after:
|
||||||
type: string
|
type: string
|
||||||
enum:
|
enum:
|
||||||
|
- step
|
||||||
- action
|
- action
|
||||||
- repository
|
- repository
|
||||||
- configuration
|
- configuration
|
||||||
@@ -1212,6 +1228,8 @@ properties:
|
|||||||
Name for the point in borgmatic's execution that
|
Name for the point in borgmatic's execution that
|
||||||
the commands should be run after (required if
|
the commands should be run after (required if
|
||||||
"before" isn't set):
|
"before" isn't set):
|
||||||
|
* "step" runs before a sub-action step for each
|
||||||
|
repository, e.g. for an individual check.
|
||||||
* "action" runs after each action for each
|
* "action" runs after each action for each
|
||||||
repository.
|
repository.
|
||||||
* "repository" runs after all actions for each
|
* "repository" runs after all actions for each
|
||||||
@@ -1254,6 +1272,18 @@ properties:
|
|||||||
particular actions listed here. Defaults to
|
particular actions listed here. Defaults to
|
||||||
running for all actions.
|
running for all actions.
|
||||||
example: [create, prune, compact, check]
|
example: [create, prune, compact, check]
|
||||||
|
steps:
|
||||||
|
type: array
|
||||||
|
items:
|
||||||
|
type: string
|
||||||
|
enum:
|
||||||
|
- archives_repository_data
|
||||||
|
- extract
|
||||||
|
- spot
|
||||||
|
description: |
|
||||||
|
List of sub-action steps for which the commands
|
||||||
|
will be run. Defaults to running for all steps.
|
||||||
|
example: [extract, spot]
|
||||||
states:
|
states:
|
||||||
type: array
|
type: array
|
||||||
items:
|
items:
|
||||||
@@ -3002,6 +3032,25 @@ properties:
|
|||||||
description: |
|
description: |
|
||||||
Configuration for integration with Linux LVM (Logical Volume
|
Configuration for integration with Linux LVM (Logical Volume
|
||||||
Manager).
|
Manager).
|
||||||
|
systemd:
|
||||||
|
type: object
|
||||||
|
additionalProperties: false
|
||||||
|
properties:
|
||||||
|
systemd_creds_command:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Command to use instead of "systemd-creds". Only used as a
|
||||||
|
fallback when borgmatic is run outside of a systemd service.
|
||||||
|
example: /usr/local/bin/systemd-creds
|
||||||
|
encrypted_credentials_directory:
|
||||||
|
type: string
|
||||||
|
description: |
|
||||||
|
Directory containing encrypted credentials for
|
||||||
|
"systemd-creds" to use instead of
|
||||||
|
"/etc/credstore.encrypted".
|
||||||
|
example: /path/to/credstore.encrypted
|
||||||
|
description: |
|
||||||
|
Configuration for integration with systemd credentials.
|
||||||
container:
|
container:
|
||||||
type: object
|
type: object
|
||||||
additionalProperties: false
|
additionalProperties: false
|
||||||
|
|||||||
@@ -64,22 +64,27 @@ def make_environment(current_environment, sys_module=sys):
|
|||||||
return environment
|
return environment
|
||||||
|
|
||||||
|
|
||||||
def filter_hooks(command_hooks, before=None, after=None, action_names=None, state_names=None):
|
def filter_hooks(command_hooks, before=None, after=None, action_names=None, step_names=None, state_names=None):
|
||||||
'''
|
'''
|
||||||
Given a sequence of command hook dicts from configuration and one or more filters (before name,
|
Given a sequence of command hook dicts from configuration and one or more filters (before name,
|
||||||
after name, a sequence of action names, and/or a sequence of execution result state names),
|
after name, a sequence of action names, a sequence of sub-action steps, and/or a sequence of
|
||||||
filter down the command hooks to just the ones that match the given filters.
|
execution result state names), filter down the command hooks to just the ones that match the
|
||||||
|
given filters.
|
||||||
'''
|
'''
|
||||||
return tuple(
|
return tuple(
|
||||||
hook_config
|
hook_config
|
||||||
for hook_config in command_hooks or ()
|
for hook_config in command_hooks or ()
|
||||||
for config_action_names in (hook_config.get('when'),)
|
for config_action_names in (hook_config.get('when'),)
|
||||||
|
for config_step_names in (hook_config.get('steps'),)
|
||||||
for config_state_names in (hook_config.get('states'),)
|
for config_state_names in (hook_config.get('states'),)
|
||||||
if before is None or hook_config.get('before') == before
|
if before is None or hook_config.get('before') == before
|
||||||
if after is None or hook_config.get('after') == after
|
if after is None or hook_config.get('after') == after
|
||||||
if action_names is None
|
if action_names is None
|
||||||
or config_action_names is None
|
or config_action_names is None
|
||||||
or set(config_action_names or ()).intersection(set(action_names))
|
or set(config_action_names or ()).intersection(set(action_names))
|
||||||
|
if step_names is None
|
||||||
|
or config_step_names is None
|
||||||
|
or set(config_step_names or ()).intersection(set(step_names))
|
||||||
if state_names is None
|
if state_names is None
|
||||||
or config_state_names is None
|
or config_state_names is None
|
||||||
or set(config_state_names or ()).intersection(set(state_names))
|
or set(config_state_names or ()).intersection(set(state_names))
|
||||||
@@ -164,7 +169,8 @@ class Before_after_hooks:
|
|||||||
before_after='do_stuff',
|
before_after='do_stuff',
|
||||||
umask=config.get('umask'),
|
umask=config.get('umask'),
|
||||||
dry_run=dry_run,
|
dry_run=dry_run,
|
||||||
action_names=['create'],
|
action_names=['check'],
|
||||||
|
step_names=['spot'],
|
||||||
):
|
):
|
||||||
do()
|
do()
|
||||||
some()
|
some()
|
||||||
@@ -182,13 +188,14 @@ class Before_after_hooks:
|
|||||||
working_directory,
|
working_directory,
|
||||||
dry_run,
|
dry_run,
|
||||||
action_names=None,
|
action_names=None,
|
||||||
|
step_names=None,
|
||||||
**context,
|
**context,
|
||||||
):
|
):
|
||||||
'''
|
'''
|
||||||
Given a sequence of command hook configuration dicts, the before/after name, a umask to run
|
Given a sequence of command hook configuration dicts, the before/after name, a umask to run
|
||||||
commands with, a working directory to run commands with, a dry run flag, a sequence of
|
commands with, a working directory to run commands with, a dry run flag, a sequence of
|
||||||
action names, and any context for the executed commands, save those data points for use
|
action names, a sequence of sub-action step names, and any context for the executed
|
||||||
below.
|
commands, save those data points for use below.
|
||||||
'''
|
'''
|
||||||
self.command_hooks = command_hooks
|
self.command_hooks = command_hooks
|
||||||
self.before_after = before_after
|
self.before_after = before_after
|
||||||
@@ -196,6 +203,7 @@ class Before_after_hooks:
|
|||||||
self.working_directory = working_directory
|
self.working_directory = working_directory
|
||||||
self.dry_run = dry_run
|
self.dry_run = dry_run
|
||||||
self.action_names = action_names
|
self.action_names = action_names
|
||||||
|
self.step_names = step_names
|
||||||
self.context = context
|
self.context = context
|
||||||
|
|
||||||
def __enter__(self):
|
def __enter__(self):
|
||||||
@@ -208,6 +216,7 @@ class Before_after_hooks:
|
|||||||
self.command_hooks,
|
self.command_hooks,
|
||||||
before=self.before_after,
|
before=self.before_after,
|
||||||
action_names=self.action_names,
|
action_names=self.action_names,
|
||||||
|
step_names=self.step_names,
|
||||||
),
|
),
|
||||||
self.umask,
|
self.umask,
|
||||||
self.working_directory,
|
self.working_directory,
|
||||||
@@ -234,6 +243,7 @@ class Before_after_hooks:
|
|||||||
self.command_hooks,
|
self.command_hooks,
|
||||||
after=self.before_after,
|
after=self.before_after,
|
||||||
action_names=self.action_names,
|
action_names=self.action_names,
|
||||||
|
step_names=self.step_names,
|
||||||
state_names=['fail' if exception_type else 'finish'],
|
state_names=['fail' if exception_type else 'finish'],
|
||||||
),
|
),
|
||||||
self.umask,
|
self.umask,
|
||||||
|
|||||||
@@ -1,6 +1,9 @@
|
|||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
import shlex
|
||||||
|
|
||||||
|
import borgmatic.execute
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -24,15 +27,28 @@ def load_credential(hook_config, config, credential_parameters):
|
|||||||
|
|
||||||
raise ValueError(f'Cannot load invalid credential name: "{name}"')
|
raise ValueError(f'Cannot load invalid credential name: "{name}"')
|
||||||
|
|
||||||
|
if not CREDENTIAL_NAME_PATTERN.match(credential_name):
|
||||||
|
raise ValueError(f'Cannot load invalid credential name "{credential_name}"')
|
||||||
|
|
||||||
credentials_directory = os.environ.get('CREDENTIALS_DIRECTORY')
|
credentials_directory = os.environ.get('CREDENTIALS_DIRECTORY')
|
||||||
|
|
||||||
if not credentials_directory:
|
if not credentials_directory:
|
||||||
raise ValueError(
|
logger.debug(
|
||||||
f'Cannot load credential "{credential_name}" because the systemd CREDENTIALS_DIRECTORY environment variable is not set',
|
f'Falling back to loading credential "{credential_name}" via systemd-creds because the systemd CREDENTIALS_DIRECTORY environment variable is not set'
|
||||||
)
|
)
|
||||||
|
|
||||||
if not CREDENTIAL_NAME_PATTERN.match(credential_name):
|
command = (
|
||||||
raise ValueError(f'Cannot load invalid credential name "{credential_name}"')
|
*shlex.split((hook_config or {}).get('systemd_creds_command', 'systemd-creds')),
|
||||||
|
'decrypt',
|
||||||
|
os.path.join(
|
||||||
|
(hook_config or {}).get(
|
||||||
|
'encrypted_credentials_directory', '/etc/credstore.encrypted'
|
||||||
|
),
|
||||||
|
credential_name,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
return borgmatic.execute.execute_command_and_capture_output(command).rstrip(os.linesep)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with open(
|
with open(
|
||||||
|
|||||||
@@ -265,7 +265,10 @@ def make_borg_snapshot_pattern(subvolume_path, pattern):
|
|||||||
rewritten_path = initial_caret + os.path.join(
|
rewritten_path = initial_caret + os.path.join(
|
||||||
subvolume_path,
|
subvolume_path,
|
||||||
f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}',
|
f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}',
|
||||||
'.', # Borg 1.4+ "slashdot" hack.
|
# Use the Borg 1.4+ "slashdot" hack to prevent the snapshot path prefix from getting
|
||||||
|
# included in the archive—but only if there's not already a slashdot hack present in the
|
||||||
|
# pattern.
|
||||||
|
('' if f'{os.path.sep}.{os.path.sep}' in pattern.path else '.'),
|
||||||
# Included so that the source directory ends up in the Borg archive at its "original" path.
|
# Included so that the source directory ends up in the Borg archive at its "original" path.
|
||||||
pattern.path.lstrip('^').lstrip(os.path.sep),
|
pattern.path.lstrip('^').lstrip(os.path.sep),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -119,10 +119,11 @@ def convert_glob_patterns_to_borg_pattern(patterns):
|
|||||||
Convert a sequence of shell glob patterns like "/etc/*", "/tmp/*" to the corresponding Borg
|
Convert a sequence of shell glob patterns like "/etc/*", "/tmp/*" to the corresponding Borg
|
||||||
regular expression archive pattern as a single string like "re:etc/.*|tmp/.*".
|
regular expression archive pattern as a single string like "re:etc/.*|tmp/.*".
|
||||||
'''
|
'''
|
||||||
# Remove the "\Z" generated by fnmatch.translate() because we don't want the pattern to match
|
# Remove the "\z" or "\Z" generated by fnmatch.translate() because we don't want the pattern to
|
||||||
# only at the end of a path, as directory format dumps require extracting files with paths
|
# match only at the end of a path, as directory format dumps require extracting files with paths
|
||||||
# longer than the pattern. E.g., a pattern of "borgmatic/*/foo_databases/test" should also match
|
# longer than the pattern. E.g., a pattern of "borgmatic/*/foo_databases/test" should also match
|
||||||
# paths like "borgmatic/*/foo_databases/test/toc.dat"
|
# paths like "borgmatic/*/foo_databases/test/toc.dat"
|
||||||
return 're:' + '|'.join(
|
return 're:' + '|'.join(
|
||||||
fnmatch.translate(pattern.lstrip('/')).replace('\\Z', '') for pattern in patterns
|
fnmatch.translate(pattern.lstrip('/')).replace('\\z', '').replace('\\Z', '')
|
||||||
|
for pattern in patterns
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -166,7 +166,10 @@ def make_borg_snapshot_pattern(pattern, logical_volume, normalized_runtime_direc
|
|||||||
hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest(
|
hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest(
|
||||||
MOUNT_POINT_HASH_LENGTH,
|
MOUNT_POINT_HASH_LENGTH,
|
||||||
),
|
),
|
||||||
'.', # Borg 1.4+ "slashdot" hack.
|
# Use the Borg 1.4+ "slashdot" hack to prevent the snapshot path prefix from getting
|
||||||
|
# included in the archive—but only if there's not already a slashdot hack present in the
|
||||||
|
# pattern.
|
||||||
|
('' if f'{os.path.sep}.{os.path.sep}' in pattern.path else '.'),
|
||||||
# Included so that the source directory ends up in the Borg archive at its "original" path.
|
# Included so that the source directory ends up in the Borg archive at its "original" path.
|
||||||
pattern.path.lstrip('^').lstrip(os.path.sep),
|
pattern.path.lstrip('^').lstrip(os.path.sep),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -214,7 +214,10 @@ def make_borg_snapshot_pattern(pattern, dataset, normalized_runtime_directory):
|
|||||||
# For instance, without this, snapshotting a dataset at /var and another at /var/spool would
|
# For instance, without this, snapshotting a dataset at /var and another at /var/spool would
|
||||||
# result in overlapping snapshot patterns and therefore colliding mount attempts.
|
# result in overlapping snapshot patterns and therefore colliding mount attempts.
|
||||||
hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest(MOUNT_POINT_HASH_LENGTH),
|
hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest(MOUNT_POINT_HASH_LENGTH),
|
||||||
'.', # Borg 1.4+ "slashdot" hack.
|
# Use the Borg 1.4+ "slashdot" hack to prevent the snapshot path prefix from getting
|
||||||
|
# included in the archive—but only if there's not already a slashdot hack present in the
|
||||||
|
# pattern.
|
||||||
|
('' if f'{os.path.sep}.{os.path.sep}' in pattern.path else '.'),
|
||||||
# Included so that the source directory ends up in the Borg archive at its "original" path.
|
# Included so that the source directory ends up in the Borg archive at its "original" path.
|
||||||
pattern.path.lstrip('^').lstrip(os.path.sep),
|
pattern.path.lstrip('^').lstrip(os.path.sep),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -77,12 +77,14 @@ commands:
|
|||||||
Each command in the `commands:` list has the following options:
|
Each command in the `commands:` list has the following options:
|
||||||
|
|
||||||
* `before` or `after`: Name for the point in borgmatic's execution that the commands should be run before or after, one of:
|
* `before` or `after`: Name for the point in borgmatic's execution that the commands should be run before or after, one of:
|
||||||
* `action` runs before each action for each repository. This replaces the deprecated `before_create`, `after_prune`, etc.
|
* <span class="minilink minilink-addedin">New in version 2.0.9</span> `step` runs before or after each sub-action step for each repository, e.g. for an individual check.
|
||||||
|
* `action` runs before or after each action for each repository. This replaces the deprecated `before_create`, `after_prune`, etc.
|
||||||
* `repository` runs before or after all actions for each repository. This replaces the deprecated `before_actions` and `after_actions`.
|
* `repository` runs before or after all actions for each repository. This replaces the deprecated `before_actions` and `after_actions`.
|
||||||
* `configuration` runs before or after all actions and repositories in the current configuration file.
|
* `configuration` runs before or after all actions and repositories in the current configuration file.
|
||||||
* `everything` runs before or after all configuration files. Errors here do not trigger `error` hooks or the `fail` state in monitoring hooks. This replaces the deprecated `before_everything` and `after_everything`.
|
* `everything` runs before or after all configuration files. Errors here do not trigger `error` hooks or the `fail` state in monitoring hooks. This replaces the deprecated `before_everything` and `after_everything`.
|
||||||
* `error` runs after an error occurs—and it's only available for `after`. This replaces the deprecated `on_error` hook.
|
* `error` runs after an error occurs—and it's only available for `after`. This replaces the deprecated `on_error` hook.
|
||||||
* `when`: Only trigger the hook when borgmatic is run with particular actions (`create`, `prune`, etc.) listed here. Defaults to running for all actions.
|
* `when`: Only trigger the hook when borgmatic is run with particular actions (`create`, `prune`, etc.) listed here. Defaults to running for all actions.
|
||||||
|
* `steps`: <span class="minilink minilink-addedin">New in version 2.0.9</span> Only trigger the hook when borgmatic runs particular sub-action steps (`extract`, `spot`, etc.) listed here. Defaults to running for all steps.
|
||||||
* `states`: <span class="minilink minilink-addedin">New in version 2.0.3</span> Only trigger the hook if borgmatic encounters one of the states (execution results) listed here. This state is evaluated only for the scope of the configured `action`, `repository`, etc., rather than for the entire borgmatic run. Only available for `after` hooks. Defaults to running the hook for all states. One or more of:
|
* `states`: <span class="minilink minilink-addedin">New in version 2.0.3</span> Only trigger the hook if borgmatic encounters one of the states (execution results) listed here. This state is evaluated only for the scope of the configured `action`, `repository`, etc., rather than for the entire borgmatic run. Only available for `after` hooks. Defaults to running the hook for all states. One or more of:
|
||||||
* `finish`: No errors occurred.
|
* `finish`: No errors occurred.
|
||||||
* `fail`: An error occurred.
|
* `fail`: An error occurred.
|
||||||
@@ -107,17 +109,22 @@ execution.
|
|||||||
|
|
||||||
Let's say you've got a borgmatic configuration file with a configured
|
Let's say you've got a borgmatic configuration file with a configured
|
||||||
repository. And suppose you configure several command hooks and then run
|
repository. And suppose you configure several command hooks and then run
|
||||||
borgmatic for the `create` and `prune` actions. Here's the order of execution:
|
borgmatic for the `create` and `check` actions. Here's the order of execution:
|
||||||
|
|
||||||
* Run `before: everything` hooks (from all configuration files).
|
* Run `before: everything` hooks (from all configuration files).
|
||||||
* Run `before: configuration` hooks (from the first configuration file).
|
* Run `before: configuration` hooks (from the first configuration file).
|
||||||
* Run `before: repository` hooks (for the first repository).
|
* Run `before: repository` hooks (for the first repository).
|
||||||
* Run `before: action` hooks for `create`.
|
* Run `before: action` hooks for `create`.
|
||||||
* Actually run the `create` action (e.g. `borg create`).
|
* Run the `create` action including `borg create`.
|
||||||
* Run `after: action` hooks for `create`.
|
* Run `after: action` hooks for `create`.
|
||||||
* Run `before: action` hooks for `prune`.
|
* Run `before: action` hooks for `check`.
|
||||||
* Actually run the `prune` action (e.g. `borg prune`).
|
* Run `before: step` hooks for the `archives_repository_data` step.
|
||||||
* Run `after: action` hooks for `prune`.
|
* Run the `borg check` portion of the `check` action.
|
||||||
|
* Run `after: step` hooks for the `archives_repository_data` step.
|
||||||
|
* Run `before: step` hooks for the `spot` step.
|
||||||
|
* Run the `spot` check portion of the `check` action.
|
||||||
|
* Run `after: step` hooks for the `spot` step.
|
||||||
|
* Run `after: action` hooks for `check`.
|
||||||
* Run `after: repository` hooks (for the first repository).
|
* Run `after: repository` hooks (for the first repository).
|
||||||
* Run `after: configuration` hooks (from the first configuration file).
|
* Run `after: configuration` hooks (from the first configuration file).
|
||||||
* Run `after: error` hooks (if an error occurs).
|
* Run `after: error` hooks (if an error occurs).
|
||||||
@@ -134,9 +141,9 @@ have a chance to run. Whereas the `after: error` hook doesn't run until all
|
|||||||
actions for—and repositories in—a configuration file have had a chance to
|
actions for—and repositories in—a configuration file have had a chance to
|
||||||
execute.
|
execute.
|
||||||
|
|
||||||
And if there are multiple hooks defined for a particular step (e.g. `before:
|
And if there are multiple hooks defined for a particular combination (e.g.
|
||||||
action` for `create`), then those hooks are run in the order they're defined in
|
`before: action` for `create`), then those hooks are run in the order they're
|
||||||
configuration.
|
defined in configuration.
|
||||||
|
|
||||||
|
|
||||||
### Deprecated command hooks
|
### Deprecated command hooks
|
||||||
|
|||||||
@@ -46,6 +46,14 @@ To get oriented with the borgmatic source code, have a look at the [source
|
|||||||
code reference](https://torsion.org/borgmatic/docs/reference/source-code/).
|
code reference](https://torsion.org/borgmatic/docs/reference/source-code/).
|
||||||
|
|
||||||
|
|
||||||
|
### Source packages
|
||||||
|
|
||||||
|
Each [borgmatic
|
||||||
|
release](https://projects.torsion.org/borgmatic-collective/borgmatic/releases)
|
||||||
|
also has source packages available. These include automated tests and serve as
|
||||||
|
a good starting point for creating third-party borgmatic packages.
|
||||||
|
|
||||||
|
|
||||||
## Automated tests
|
## Automated tests
|
||||||
|
|
||||||
Assuming you've cloned the borgmatic source code as described above and you're
|
Assuming you've cloned the borgmatic source code as described above and you're
|
||||||
|
|||||||
@@ -127,15 +127,26 @@ encryption_passcommand: cat ${CREDENTIALS_DIRECTORY}/borgmatic_backupserver1
|
|||||||
Adjust `borgmatic_backupserver1` according to the name of the credential and the
|
Adjust `borgmatic_backupserver1` according to the name of the credential and the
|
||||||
directory set in the service file.
|
directory set in the service file.
|
||||||
|
|
||||||
Be aware that when using this systemd `{credential ...}` feature, you may no
|
<span class="minilink minilink-addedin">New in version 2.0.9</span> When using
|
||||||
longer be able to run certain borgmatic actions outside of the systemd service,
|
the systemd `{credential ...}` feature, borgmatic loads systemd credentials even
|
||||||
as the credentials are only available from within the context of that service.
|
when run outside of a systemd service. This works by falling back to calling
|
||||||
So for instance, `borgmatic list` necessarily relies on the
|
`systemd-creds decrypt` instead of reading credentials directly. To customize
|
||||||
`encryption_passphrase` in order to access the Borg repository, but `list`
|
this behavior, you can override the `systemd-creds` command and/or the
|
||||||
shouldn't need to load any credentials for your database or monitoring hooks.
|
credential store directory it uses:
|
||||||
|
|
||||||
The one exception is `borgmatic config validate`, which doesn't actually load
|
```yaml
|
||||||
any credentials and should continue working anywhere.
|
systemd:
|
||||||
|
systemd_creds_command: /usr/local/bin/systemd-creds
|
||||||
|
encrypted_credentials_directory: /path/to/credstore.encrypted
|
||||||
|
```
|
||||||
|
|
||||||
|
<span class="minilink minilink-addedin">Prior to version 2.0.9</span> The
|
||||||
|
systemd `{credential ...}` feature did not work when run outside of a systemd
|
||||||
|
service. But depending on the borgmatic action invoked and the configuration
|
||||||
|
option where `{credential ...}` was used, you could sometimes get away without
|
||||||
|
working systemd credentials for certain actions. For instance, `borgmatic list`
|
||||||
|
doesn't connect to any databases or monitoring services, and `borgmatic config
|
||||||
|
validate` doesn't use credentials as all.
|
||||||
|
|
||||||
|
|
||||||
### Container secrets
|
### Container secrets
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "borgmatic"
|
name = "borgmatic"
|
||||||
version = "2.0.8"
|
version = "2.0.9.dev0"
|
||||||
authors = [
|
authors = [
|
||||||
{ name="Dan Helfman", email="witten@torsion.org" },
|
{ name="Dan Helfman", email="witten@torsion.org" },
|
||||||
]
|
]
|
||||||
|
|||||||
+1
-1
@@ -33,7 +33,7 @@ git push github $version
|
|||||||
|
|
||||||
# Build borgmatic and publish to pypi.
|
# Build borgmatic and publish to pypi.
|
||||||
rm -fr dist
|
rm -fr dist
|
||||||
python3 -m build
|
uv build
|
||||||
twine upload -r pypi --username __token__ dist/borgmatic-*.tar.gz
|
twine upload -r pypi --username __token__ dist/borgmatic-*.tar.gz
|
||||||
twine upload -r pypi --username __token__ dist/borgmatic-*-py3-none-any.whl
|
twine upload -r pypi --username __token__ dist/borgmatic-*-py3-none-any.whl
|
||||||
|
|
||||||
|
|||||||
@@ -19,13 +19,60 @@ def test_load_credential_with_invalid_credential_parameters_raises(credential_pa
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_load_credential_without_credentials_directory_raises():
|
def test_load_credential_without_credentials_directory_falls_back_to_systemd_creds_command():
|
||||||
flexmock(module.os.environ).should_receive('get').with_args('CREDENTIALS_DIRECTORY').and_return(
|
flexmock(module.os.environ).should_receive('get').with_args('CREDENTIALS_DIRECTORY').and_return(
|
||||||
None,
|
None,
|
||||||
)
|
)
|
||||||
|
flexmock(module.borgmatic.execute).should_receive(
|
||||||
|
'execute_command_and_capture_output'
|
||||||
|
).with_args(('systemd-creds', 'decrypt', '/etc/credstore.encrypted/mycredential')).and_return(
|
||||||
|
'password'
|
||||||
|
).once()
|
||||||
|
|
||||||
with pytest.raises(ValueError):
|
assert (
|
||||||
module.load_credential(hook_config={}, config={}, credential_parameters=('mycredential',))
|
module.load_credential(hook_config={}, config={}, credential_parameters=('mycredential',))
|
||||||
|
== 'password'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_credential_without_credentials_directory_calls_custom_systemd_creds_command():
|
||||||
|
flexmock(module.os.environ).should_receive('get').with_args('CREDENTIALS_DIRECTORY').and_return(
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
flexmock(module.borgmatic.execute).should_receive(
|
||||||
|
'execute_command_and_capture_output'
|
||||||
|
).with_args(
|
||||||
|
('/path/to/systemd-creds', '--flag', 'decrypt', '/etc/credstore.encrypted/mycredential')
|
||||||
|
).and_return('password').once()
|
||||||
|
|
||||||
|
assert (
|
||||||
|
module.load_credential(
|
||||||
|
hook_config={'systemd_creds_command': '/path/to/systemd-creds --flag'},
|
||||||
|
config={},
|
||||||
|
credential_parameters=('mycredential',),
|
||||||
|
)
|
||||||
|
== 'password'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_credential_without_credentials_directory_uses_custom_encrypted_credentials_directory():
|
||||||
|
flexmock(module.os.environ).should_receive('get').with_args('CREDENTIALS_DIRECTORY').and_return(
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
flexmock(module.borgmatic.execute).should_receive(
|
||||||
|
'execute_command_and_capture_output'
|
||||||
|
).with_args(('systemd-creds', 'decrypt', '/my/credstore.encrypted/mycredential')).and_return(
|
||||||
|
'password'
|
||||||
|
).once()
|
||||||
|
|
||||||
|
assert (
|
||||||
|
module.load_credential(
|
||||||
|
hook_config={'encrypted_credentials_directory': '/my/credstore.encrypted'},
|
||||||
|
config={},
|
||||||
|
credential_parameters=('mycredential',),
|
||||||
|
)
|
||||||
|
== 'password'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_load_credential_with_invalid_credential_name_raises():
|
def test_load_credential_with_invalid_credential_name_raises():
|
||||||
|
|||||||
@@ -326,6 +326,11 @@ def test_make_snapshot_path_includes_stripped_subvolume_path(
|
|||||||
),
|
),
|
||||||
('/', Pattern('/foo'), Pattern('/.borgmatic-snapshot-1234/./foo')),
|
('/', Pattern('/foo'), Pattern('/.borgmatic-snapshot-1234/./foo')),
|
||||||
('/', Pattern('/'), Pattern('/.borgmatic-snapshot-1234/./')),
|
('/', Pattern('/'), Pattern('/.borgmatic-snapshot-1234/./')),
|
||||||
|
(
|
||||||
|
'/foo/bar',
|
||||||
|
Pattern('/foo/bar/./baz'),
|
||||||
|
Pattern('/foo/bar/.borgmatic-snapshot-1234/foo/bar/./baz'),
|
||||||
|
),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path(
|
def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path(
|
||||||
|
|||||||
@@ -270,6 +270,10 @@ def test_snapshot_logical_volume_with_non_percentage_snapshot_name_uses_lvcreate
|
|||||||
),
|
),
|
||||||
(Pattern('/foo'), Pattern('/run/borgmatic/lvm_snapshots/b33f/./foo')),
|
(Pattern('/foo'), Pattern('/run/borgmatic/lvm_snapshots/b33f/./foo')),
|
||||||
(Pattern('/'), Pattern('/run/borgmatic/lvm_snapshots/b33f/./')),
|
(Pattern('/'), Pattern('/run/borgmatic/lvm_snapshots/b33f/./')),
|
||||||
|
(
|
||||||
|
Pattern('/foo/./bar/baz'),
|
||||||
|
Pattern('/run/borgmatic/lvm_snapshots/b33f/foo/./bar/baz'),
|
||||||
|
),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path(
|
def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path(
|
||||||
|
|||||||
@@ -278,6 +278,10 @@ def test_get_all_dataset_mount_points_omits_duplicates():
|
|||||||
),
|
),
|
||||||
(Pattern('/foo'), Pattern('/run/borgmatic/zfs_snapshots/b33f/./foo')),
|
(Pattern('/foo'), Pattern('/run/borgmatic/zfs_snapshots/b33f/./foo')),
|
||||||
(Pattern('/'), Pattern('/run/borgmatic/zfs_snapshots/b33f/./')),
|
(Pattern('/'), Pattern('/run/borgmatic/zfs_snapshots/b33f/./')),
|
||||||
|
(
|
||||||
|
Pattern('/foo/./bar/baz'),
|
||||||
|
Pattern('/run/borgmatic/zfs_snapshots/b33f/foo/./bar/baz'),
|
||||||
|
),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path(
|
def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path(
|
||||||
|
|||||||
Reference in New Issue
Block a user