Compare commits

..
21 changed files with 288 additions and 69 deletions
+6 -1
View File
@@ -1,2 +1,7 @@
# This file only applies to the source dist tarball, not the built wheel.
include borgmatic/config/schema.yaml
graft sample/systemd
graft docs
graft sample
graft scripts
graft tests
global-exclude *.py[co]
+10
View File
@@ -1,3 +1,13 @@
2.0.9.dev0
* #1123: Add loading of systemd credentials even when running borgmatic outside of a systemd
service.
* #1134: Add a "steps" option to run command hooks around particular sub-action steps like
individual checks.
* #1149: Add support for Python 3.14.
* #1149: Include automated tests in the source dist tarball uploaded to PyPI.
* #1151: Fix snapshotting in the ZFS, Btrfs, and LVM hooks to play nicely with the Borg 1.4+
"slashdot" hack within source directory paths.
2.0.8
* #1114: Document systemd configuration changes for the ZFS filesystem hook.
* #1116: Add dumping of database containers via their container names, handy for backing up
+62 -30
View File
@@ -744,6 +744,7 @@ def run_check(
global_arguments,
local_path,
remote_path,
hook_context,
):
'''
Run the "check" action for the given repository.
@@ -783,44 +784,75 @@ def run_check(
archives_check_id,
)
borg_specific_checks = set(checks).intersection({'repository', 'archives', 'data'})
working_directory = borgmatic.config.paths.get_working_directory(config)
if borg_specific_checks:
borgmatic.borg.check.check_archives(
repository['path'],
config,
local_borg_version,
check_arguments,
global_arguments,
borg_specific_checks,
archive_filter_flags,
local_path=local_path,
remote_path=remote_path,
)
for check in borg_specific_checks:
write_check_time(make_check_time_path(config, repository_id, check, archives_check_id))
with borgmatic.hooks.command.Before_after_hooks(
command_hooks=config.get('commands'),
before_after='step',
umask=config.get('umask'),
working_directory=working_directory,
dry_run=global_arguments.dry_run,
action_names=('check',),
step_names=('archives_repository_data',),
**hook_context,
):
borgmatic.borg.check.check_archives(
repository['path'],
config,
local_borg_version,
check_arguments,
global_arguments,
borg_specific_checks,
archive_filter_flags,
local_path=local_path,
remote_path=remote_path,
)
for check in borg_specific_checks:
write_check_time(make_check_time_path(config, repository_id, check, archives_check_id))
if 'extract' in checks:
borgmatic.borg.extract.extract_last_archive_dry_run(
config,
local_borg_version,
global_arguments,
repository['path'],
config.get('lock_wait'),
local_path,
remote_path,
)
write_check_time(make_check_time_path(config, repository_id, 'extract'))
if 'spot' in checks:
with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
spot_check(
repository,
with borgmatic.hooks.command.Before_after_hooks(
command_hooks=config.get('commands'),
before_after='step',
umask=config.get('umask'),
working_directory=working_directory,
dry_run=global_arguments.dry_run,
action_names=('check',),
step_names=('extract',),
**hook_context,
):
borgmatic.borg.extract.extract_last_archive_dry_run(
config,
local_borg_version,
global_arguments,
repository['path'],
config.get('lock_wait'),
local_path,
remote_path,
borgmatic_runtime_directory,
)
write_check_time(make_check_time_path(config, repository_id, 'extract'))
write_check_time(make_check_time_path(config, repository_id, 'spot'))
if 'spot' in checks:
with borgmatic.hooks.command.Before_after_hooks(
command_hooks=config.get('commands'),
before_after='step',
umask=config.get('umask'),
working_directory=working_directory,
dry_run=global_arguments.dry_run,
action_names=('check',),
step_names=('spot',),
**hook_context,
):
with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
spot_check(
repository,
config,
local_borg_version,
global_arguments,
local_path,
remote_path,
borgmatic_runtime_directory,
)
write_check_time(make_check_time_path(config, repository_id, 'spot'))
+1
View File
@@ -463,6 +463,7 @@ def run_actions( # noqa: PLR0912, PLR0915
global_arguments,
local_path,
remote_path,
hook_context,
)
elif action_name == 'extract':
borgmatic.actions.extract.run_extract(
+49
View File
@@ -1140,6 +1140,7 @@ properties:
before:
type: string
enum:
- step
- action
- repository
- configuration
@@ -1148,6 +1149,8 @@ properties:
Name for the point in borgmatic's execution that
the commands should be run before (required if
"after" isn't set):
* "step" runs before a sub-action step for each
repository, e.g. for an individual check.
* "action" runs before each action for each
repository.
* "repository" runs before all actions for each
@@ -1188,6 +1191,18 @@ properties:
List of actions for which the commands will be
run. Defaults to running for all actions.
example: [create, prune, compact, check]
steps:
type: array
items:
type: string
enum:
- archives_repository_data
- extract
- spot
description: |
List of sub-action steps for which the commands
will be run. Defaults to running for all steps.
example: [extract, spot]
run:
type: array
items:
@@ -1203,6 +1218,7 @@ properties:
after:
type: string
enum:
- step
- action
- repository
- configuration
@@ -1212,6 +1228,8 @@ properties:
Name for the point in borgmatic's execution that
the commands should be run after (required if
"before" isn't set):
* "step" runs before a sub-action step for each
repository, e.g. for an individual check.
* "action" runs after each action for each
repository.
* "repository" runs after all actions for each
@@ -1254,6 +1272,18 @@ properties:
particular actions listed here. Defaults to
running for all actions.
example: [create, prune, compact, check]
steps:
type: array
items:
type: string
enum:
- archives_repository_data
- extract
- spot
description: |
List of sub-action steps for which the commands
will be run. Defaults to running for all steps.
example: [extract, spot]
states:
type: array
items:
@@ -3002,6 +3032,25 @@ properties:
description: |
Configuration for integration with Linux LVM (Logical Volume
Manager).
systemd:
type: object
additionalProperties: false
properties:
systemd_creds_command:
type: string
description: |
Command to use instead of "systemd-creds". Only used as a
fallback when borgmatic is run outside of a systemd service.
example: /usr/local/bin/systemd-creds
encrypted_credentials_directory:
type: string
description: |
Directory containing encrypted credentials for
"systemd-creds" to use instead of
"/etc/credstore.encrypted".
example: /path/to/credstore.encrypted
description: |
Configuration for integration with systemd credentials.
container:
type: object
additionalProperties: false
+16 -6
View File
@@ -64,22 +64,27 @@ def make_environment(current_environment, sys_module=sys):
return environment
def filter_hooks(command_hooks, before=None, after=None, action_names=None, state_names=None):
def filter_hooks(command_hooks, before=None, after=None, action_names=None, step_names=None, state_names=None):
'''
Given a sequence of command hook dicts from configuration and one or more filters (before name,
after name, a sequence of action names, and/or a sequence of execution result state names),
filter down the command hooks to just the ones that match the given filters.
after name, a sequence of action names, a sequence of sub-action steps, and/or a sequence of
execution result state names), filter down the command hooks to just the ones that match the
given filters.
'''
return tuple(
hook_config
for hook_config in command_hooks or ()
for config_action_names in (hook_config.get('when'),)
for config_step_names in (hook_config.get('steps'),)
for config_state_names in (hook_config.get('states'),)
if before is None or hook_config.get('before') == before
if after is None or hook_config.get('after') == after
if action_names is None
or config_action_names is None
or set(config_action_names or ()).intersection(set(action_names))
if step_names is None
or config_step_names is None
or set(config_step_names or ()).intersection(set(step_names))
if state_names is None
or config_state_names is None
or set(config_state_names or ()).intersection(set(state_names))
@@ -164,7 +169,8 @@ class Before_after_hooks:
before_after='do_stuff',
umask=config.get('umask'),
dry_run=dry_run,
action_names=['create'],
action_names=['check'],
step_names=['spot'],
):
do()
some()
@@ -182,13 +188,14 @@ class Before_after_hooks:
working_directory,
dry_run,
action_names=None,
step_names=None,
**context,
):
'''
Given a sequence of command hook configuration dicts, the before/after name, a umask to run
commands with, a working directory to run commands with, a dry run flag, a sequence of
action names, and any context for the executed commands, save those data points for use
below.
action names, a sequence of sub-action step names, and any context for the executed
commands, save those data points for use below.
'''
self.command_hooks = command_hooks
self.before_after = before_after
@@ -196,6 +203,7 @@ class Before_after_hooks:
self.working_directory = working_directory
self.dry_run = dry_run
self.action_names = action_names
self.step_names = step_names
self.context = context
def __enter__(self):
@@ -208,6 +216,7 @@ class Before_after_hooks:
self.command_hooks,
before=self.before_after,
action_names=self.action_names,
step_names=self.step_names,
),
self.umask,
self.working_directory,
@@ -234,6 +243,7 @@ class Before_after_hooks:
self.command_hooks,
after=self.before_after,
action_names=self.action_names,
step_names=self.step_names,
state_names=['fail' if exception_type else 'finish'],
),
self.umask,
+20 -4
View File
@@ -1,6 +1,9 @@
import logging
import os
import re
import shlex
import borgmatic.execute
logger = logging.getLogger(__name__)
@@ -24,15 +27,28 @@ def load_credential(hook_config, config, credential_parameters):
raise ValueError(f'Cannot load invalid credential name: "{name}"')
if not CREDENTIAL_NAME_PATTERN.match(credential_name):
raise ValueError(f'Cannot load invalid credential name "{credential_name}"')
credentials_directory = os.environ.get('CREDENTIALS_DIRECTORY')
if not credentials_directory:
raise ValueError(
f'Cannot load credential "{credential_name}" because the systemd CREDENTIALS_DIRECTORY environment variable is not set',
logger.debug(
f'Falling back to loading credential "{credential_name}" via systemd-creds because the systemd CREDENTIALS_DIRECTORY environment variable is not set'
)
if not CREDENTIAL_NAME_PATTERN.match(credential_name):
raise ValueError(f'Cannot load invalid credential name "{credential_name}"')
command = (
*shlex.split((hook_config or {}).get('systemd_creds_command', 'systemd-creds')),
'decrypt',
os.path.join(
(hook_config or {}).get(
'encrypted_credentials_directory', '/etc/credstore.encrypted'
),
credential_name,
),
)
return borgmatic.execute.execute_command_and_capture_output(command).rstrip(os.linesep)
try:
with open(
+4 -1
View File
@@ -265,7 +265,10 @@ def make_borg_snapshot_pattern(subvolume_path, pattern):
rewritten_path = initial_caret + os.path.join(
subvolume_path,
f'{BORGMATIC_SNAPSHOT_PREFIX}{os.getpid()}',
'.', # Borg 1.4+ "slashdot" hack.
# Use the Borg 1.4+ "slashdot" hack to prevent the snapshot path prefix from getting
# included in the archive—but only if there's not already a slashdot hack present in the
# pattern.
('' if f'{os.path.sep}.{os.path.sep}' in pattern.path else '.'),
# Included so that the source directory ends up in the Borg archive at its "original" path.
pattern.path.lstrip('^').lstrip(os.path.sep),
)
+4 -3
View File
@@ -119,10 +119,11 @@ def convert_glob_patterns_to_borg_pattern(patterns):
Convert a sequence of shell glob patterns like "/etc/*", "/tmp/*" to the corresponding Borg
regular expression archive pattern as a single string like "re:etc/.*|tmp/.*".
'''
# Remove the "\Z" generated by fnmatch.translate() because we don't want the pattern to match
# only at the end of a path, as directory format dumps require extracting files with paths
# Remove the "\z" or "\Z" generated by fnmatch.translate() because we don't want the pattern to
# match only at the end of a path, as directory format dumps require extracting files with paths
# longer than the pattern. E.g., a pattern of "borgmatic/*/foo_databases/test" should also match
# paths like "borgmatic/*/foo_databases/test/toc.dat"
return 're:' + '|'.join(
fnmatch.translate(pattern.lstrip('/')).replace('\\Z', '') for pattern in patterns
fnmatch.translate(pattern.lstrip('/')).replace('\\z', '').replace('\\Z', '')
for pattern in patterns
)
+4 -1
View File
@@ -166,7 +166,10 @@ def make_borg_snapshot_pattern(pattern, logical_volume, normalized_runtime_direc
hashlib.shake_256(logical_volume.mount_point.encode('utf-8')).hexdigest(
MOUNT_POINT_HASH_LENGTH,
),
'.', # Borg 1.4+ "slashdot" hack.
# Use the Borg 1.4+ "slashdot" hack to prevent the snapshot path prefix from getting
# included in the archive—but only if there's not already a slashdot hack present in the
# pattern.
('' if f'{os.path.sep}.{os.path.sep}' in pattern.path else '.'),
# Included so that the source directory ends up in the Borg archive at its "original" path.
pattern.path.lstrip('^').lstrip(os.path.sep),
)
+4 -1
View File
@@ -214,7 +214,10 @@ def make_borg_snapshot_pattern(pattern, dataset, normalized_runtime_directory):
# For instance, without this, snapshotting a dataset at /var and another at /var/spool would
# result in overlapping snapshot patterns and therefore colliding mount attempts.
hashlib.shake_256(dataset.mount_point.encode('utf-8')).hexdigest(MOUNT_POINT_HASH_LENGTH),
'.', # Borg 1.4+ "slashdot" hack.
# Use the Borg 1.4+ "slashdot" hack to prevent the snapshot path prefix from getting
# included in the archive—but only if there's not already a slashdot hack present in the
# pattern.
('' if f'{os.path.sep}.{os.path.sep}' in pattern.path else '.'),
# Included so that the source directory ends up in the Borg archive at its "original" path.
pattern.path.lstrip('^').lstrip(os.path.sep),
)
@@ -77,12 +77,14 @@ commands:
Each command in the `commands:` list has the following options:
* `before` or `after`: Name for the point in borgmatic's execution that the commands should be run before or after, one of:
* `action` runs before each action for each repository. This replaces the deprecated `before_create`, `after_prune`, etc.
* <span class="minilink minilink-addedin">New in version 2.0.9</span> `step` runs before or after each sub-action step for each repository, e.g. for an individual check.
* `action` runs before or after each action for each repository. This replaces the deprecated `before_create`, `after_prune`, etc.
* `repository` runs before or after all actions for each repository. This replaces the deprecated `before_actions` and `after_actions`.
* `configuration` runs before or after all actions and repositories in the current configuration file.
* `everything` runs before or after all configuration files. Errors here do not trigger `error` hooks or the `fail` state in monitoring hooks. This replaces the deprecated `before_everything` and `after_everything`.
* `error` runs after an error occurs—and it's only available for `after`. This replaces the deprecated `on_error` hook.
* `when`: Only trigger the hook when borgmatic is run with particular actions (`create`, `prune`, etc.) listed here. Defaults to running for all actions.
* `steps`: <span class="minilink minilink-addedin">New in version 2.0.9</span> Only trigger the hook when borgmatic runs particular sub-action steps (`extract`, `spot`, etc.) listed here. Defaults to running for all steps.
* `states`: <span class="minilink minilink-addedin">New in version 2.0.3</span> Only trigger the hook if borgmatic encounters one of the states (execution results) listed here. This state is evaluated only for the scope of the configured `action`, `repository`, etc., rather than for the entire borgmatic run. Only available for `after` hooks. Defaults to running the hook for all states. One or more of:
* `finish`: No errors occurred.
* `fail`: An error occurred.
@@ -107,17 +109,22 @@ execution.
Let's say you've got a borgmatic configuration file with a configured
repository. And suppose you configure several command hooks and then run
borgmatic for the `create` and `prune` actions. Here's the order of execution:
borgmatic for the `create` and `check` actions. Here's the order of execution:
* Run `before: everything` hooks (from all configuration files).
* Run `before: configuration` hooks (from the first configuration file).
* Run `before: repository` hooks (for the first repository).
* Run `before: action` hooks for `create`.
* Actually run the `create` action (e.g. `borg create`).
* Run the `create` action including `borg create`.
* Run `after: action` hooks for `create`.
* Run `before: action` hooks for `prune`.
* Actually run the `prune` action (e.g. `borg prune`).
* Run `after: action` hooks for `prune`.
* Run `before: action` hooks for `check`.
* Run `before: step` hooks for the `archives_repository_data` step.
* Run the `borg check` portion of the `check` action.
* Run `after: step` hooks for the `archives_repository_data` step.
* Run `before: step` hooks for the `spot` step.
* Run the `spot` check portion of the `check` action.
* Run `after: step` hooks for the `spot` step.
* Run `after: action` hooks for `check`.
* Run `after: repository` hooks (for the first repository).
* Run `after: configuration` hooks (from the first configuration file).
* Run `after: error` hooks (if an error occurs).
@@ -134,9 +141,9 @@ have a chance to run. Whereas the `after: error` hook doesn't run until all
actions for—and repositories in—a configuration file have had a chance to
execute.
And if there are multiple hooks defined for a particular step (e.g. `before:
action` for `create`), then those hooks are run in the order they're defined in
configuration.
And if there are multiple hooks defined for a particular combination (e.g.
`before: action` for `create`), then those hooks are run in the order they're
defined in configuration.
### Deprecated command hooks
+8
View File
@@ -46,6 +46,14 @@ To get oriented with the borgmatic source code, have a look at the [source
code reference](https://torsion.org/borgmatic/docs/reference/source-code/).
### Source packages
Each [borgmatic
release](https://projects.torsion.org/borgmatic-collective/borgmatic/releases)
also has source packages available. These include automated tests and serve as
a good starting point for creating third-party borgmatic packages.
## Automated tests
Assuming you've cloned the borgmatic source code as described above and you're
+19 -8
View File
@@ -127,15 +127,26 @@ encryption_passcommand: cat ${CREDENTIALS_DIRECTORY}/borgmatic_backupserver1
Adjust `borgmatic_backupserver1` according to the name of the credential and the
directory set in the service file.
Be aware that when using this systemd `{credential ...}` feature, you may no
longer be able to run certain borgmatic actions outside of the systemd service,
as the credentials are only available from within the context of that service.
So for instance, `borgmatic list` necessarily relies on the
`encryption_passphrase` in order to access the Borg repository, but `list`
shouldn't need to load any credentials for your database or monitoring hooks.
<span class="minilink minilink-addedin">New in version 2.0.9</span> When using
the systemd `{credential ...}` feature, borgmatic loads systemd credentials even
when run outside of a systemd service. This works by falling back to calling
`systemd-creds decrypt` instead of reading credentials directly. To customize
this behavior, you can override the `systemd-creds` command and/or the
credential store directory it uses:
The one exception is `borgmatic config validate`, which doesn't actually load
any credentials and should continue working anywhere.
```yaml
systemd:
systemd_creds_command: /usr/local/bin/systemd-creds
encrypted_credentials_directory: /path/to/credstore.encrypted
```
<span class="minilink minilink-addedin">Prior to version 2.0.9</span> The
systemd `{credential ...}` feature did not work when run outside of a systemd
service. But depending on the borgmatic action invoked and the configuration
option where `{credential ...}` was used, you could sometimes get away without
working systemd credentials for certain actions. For instance, `borgmatic list`
doesn't connect to any databases or monitoring services, and `borgmatic config
validate` doesn't use credentials as all.
### Container secrets
+1 -1
View File
@@ -1,6 +1,6 @@
[project]
name = "borgmatic"
version = "2.0.8"
version = "2.0.9.dev0"
authors = [
{ name="Dan Helfman", email="witten@torsion.org" },
]
+1 -1
View File
@@ -33,7 +33,7 @@ git push github $version
# Build borgmatic and publish to pypi.
rm -fr dist
python3 -m build
uv build
twine upload -r pypi --username __token__ dist/borgmatic-*.tar.gz
twine upload -r pypi --username __token__ dist/borgmatic-*-py3-none-any.whl
+49 -2
View File
@@ -19,13 +19,60 @@ def test_load_credential_with_invalid_credential_parameters_raises(credential_pa
)
def test_load_credential_without_credentials_directory_raises():
def test_load_credential_without_credentials_directory_falls_back_to_systemd_creds_command():
flexmock(module.os.environ).should_receive('get').with_args('CREDENTIALS_DIRECTORY').and_return(
None,
)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).with_args(('systemd-creds', 'decrypt', '/etc/credstore.encrypted/mycredential')).and_return(
'password'
).once()
with pytest.raises(ValueError):
assert (
module.load_credential(hook_config={}, config={}, credential_parameters=('mycredential',))
== 'password'
)
def test_load_credential_without_credentials_directory_calls_custom_systemd_creds_command():
flexmock(module.os.environ).should_receive('get').with_args('CREDENTIALS_DIRECTORY').and_return(
None,
)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).with_args(
('/path/to/systemd-creds', '--flag', 'decrypt', '/etc/credstore.encrypted/mycredential')
).and_return('password').once()
assert (
module.load_credential(
hook_config={'systemd_creds_command': '/path/to/systemd-creds --flag'},
config={},
credential_parameters=('mycredential',),
)
== 'password'
)
def test_load_credential_without_credentials_directory_uses_custom_encrypted_credentials_directory():
flexmock(module.os.environ).should_receive('get').with_args('CREDENTIALS_DIRECTORY').and_return(
None,
)
flexmock(module.borgmatic.execute).should_receive(
'execute_command_and_capture_output'
).with_args(('systemd-creds', 'decrypt', '/my/credstore.encrypted/mycredential')).and_return(
'password'
).once()
assert (
module.load_credential(
hook_config={'encrypted_credentials_directory': '/my/credstore.encrypted'},
config={},
credential_parameters=('mycredential',),
)
== 'password'
)
def test_load_credential_with_invalid_credential_name_raises():
@@ -326,6 +326,11 @@ def test_make_snapshot_path_includes_stripped_subvolume_path(
),
('/', Pattern('/foo'), Pattern('/.borgmatic-snapshot-1234/./foo')),
('/', Pattern('/'), Pattern('/.borgmatic-snapshot-1234/./')),
(
'/foo/bar',
Pattern('/foo/bar/./baz'),
Pattern('/foo/bar/.borgmatic-snapshot-1234/foo/bar/./baz'),
),
),
)
def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path(
+4
View File
@@ -270,6 +270,10 @@ def test_snapshot_logical_volume_with_non_percentage_snapshot_name_uses_lvcreate
),
(Pattern('/foo'), Pattern('/run/borgmatic/lvm_snapshots/b33f/./foo')),
(Pattern('/'), Pattern('/run/borgmatic/lvm_snapshots/b33f/./')),
(
Pattern('/foo/./bar/baz'),
Pattern('/run/borgmatic/lvm_snapshots/b33f/foo/./bar/baz'),
),
),
)
def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path(
+4
View File
@@ -278,6 +278,10 @@ def test_get_all_dataset_mount_points_omits_duplicates():
),
(Pattern('/foo'), Pattern('/run/borgmatic/zfs_snapshots/b33f/./foo')),
(Pattern('/'), Pattern('/run/borgmatic/zfs_snapshots/b33f/./')),
(
Pattern('/foo/./bar/baz'),
Pattern('/run/borgmatic/zfs_snapshots/b33f/foo/./bar/baz'),
),
),
)
def test_make_borg_snapshot_pattern_includes_slashdot_hack_and_stripped_pattern_path(
+1 -1
View File
@@ -1,5 +1,5 @@
[tox]
env_list = py39,py310,py311,py312,py313
env_list = py39,py310,py311,py312,py313,py314
skip_missing_interpreters = true
package = editable
min_version = 4.0