Compare commits

..
9 Commits
15 changed files with 491 additions and 39 deletions
+14
View File
@@ -1,3 +1,17 @@
1.4.15
* Fix for database dump removal incorrectly skipping some database dumps.
* #123: Support for mounting an archive as a FUSE filesystem via "borgmatic mount" action, and
unmounting via "borgmatic umount". See the documentation for more information:
https://torsion.org/borgmatic/docs/how-to/extract-a-backup/#mount-a-filesystem
1.4.14
* Show summary log errors regardless of verbosity level, and log the "summary:" header with a log
level based on the contained summary logs.
1.4.13
* Show full error logs at "--verbosity 0" so you can see command output without upping the
verbosity level.
1.4.12
* #247: With "borgmatic check", consider Borg warnings as errors.
* Dial back the display of inline error logs a bit, so failed command output doesn't appear
+46
View File
@@ -0,0 +1,46 @@
import logging
from borgmatic.execute import execute_command, execute_command_without_capture
logger = logging.getLogger(__name__)
def mount_archive(
repository,
archive,
mount_point,
paths,
foreground,
options,
storage_config,
local_path='borg',
remote_path=None,
):
'''
Given a local or remote repository path, an archive name, a filesystem mount point, zero or more
paths to mount from the archive, extra Borg mount options, a storage configuration dict, and
optional local and remote Borg paths, mount the archive onto the mount point.
'''
umask = storage_config.get('umask', None)
lock_wait = storage_config.get('lock_wait', None)
full_command = (
(local_path, 'mount')
+ (('--remote-path', remote_path) if remote_path else ())
+ (('--umask', str(umask)) if umask else ())
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+ (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+ (('--foreground',) if foreground else ())
+ (('-o', options) if options else ())
+ ('::'.join((repository, archive)),)
+ (mount_point,)
+ (tuple(paths) if paths else ())
)
# Don't capture the output when foreground mode is used so that ctrl-C can work properly.
if foreground:
execute_command_without_capture(full_command)
return
execute_command(full_command)
+20
View File
@@ -0,0 +1,20 @@
import logging
from borgmatic.execute import execute_command
logger = logging.getLogger(__name__)
def unmount_archive(mount_point, local_path='borg'):
'''
Given a mounted filesystem mount point, and an optional local Borg paths, umount the filesystem
from the mount point.
'''
full_command = (
(local_path, 'umount')
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
+ (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
+ (mount_point,)
)
execute_command(full_command, error_on_warnings=True)
+62 -6
View File
@@ -9,6 +9,8 @@ SUBPARSER_ALIASES = {
'create': ['--create', '-C'],
'check': ['--check', '-k'],
'extract': ['--extract', '-x'],
'mount': ['--mount', '-m'],
'umount': ['--umount', '-u'],
'restore': ['--restore', '-r'],
'list': ['--list', '-l'],
'info': ['--info', '-i'],
@@ -140,21 +142,21 @@ def parse_arguments(*unparsed_arguments):
type=int,
choices=range(-1, 3),
default=0,
help='Display verbose progress to the console (from none to lots: 0, 1, or 2) or only errors (-1)',
help='Display verbose progress to the console (from only errors to very verbose: -1, 0, 1, or 2)',
)
global_group.add_argument(
'--syslog-verbosity',
type=int,
choices=range(-1, 3),
default=0,
help='Log verbose progress to syslog (from none to lots: 0, 1, or 2) or only errors (-1). Ignored when console is interactive or --log-file is given',
help='Log verbose progress to syslog (from only errors to very verbose: -1, 0, 1, or 2). Ignored when console is interactive or --log-file is given',
)
global_group.add_argument(
'--log-file-verbosity',
type=int,
choices=range(-1, 3),
default=0,
help='Log verbose progress to log file (from none to lots: 0, 1, or 2) or only errors (-1). Only used when --log-file is given',
help='Log verbose progress to log file (from only errors to very verbose: -1, 0, 1, or 2). Only used when --log-file is given',
)
global_group.add_argument(
'--log-file',
@@ -172,9 +174,9 @@ def parse_arguments(*unparsed_arguments):
top_level_parser = ArgumentParser(
description='''
A simple wrapper script for the Borg backup software that creates and prunes backups.
If none of the action options are given, then borgmatic defaults to: prune, create, and
check archives.
Simple, configuration-driven backup software for servers and workstations. If none of
the action options are given, then borgmatic defaults to: prune, create, and check
archives.
''',
parents=[global_parser],
)
@@ -312,6 +314,60 @@ def parse_arguments(*unparsed_arguments):
'-h', '--help', action='help', help='Show this help message and exit'
)
mount_parser = subparsers.add_parser(
'mount',
aliases=SUBPARSER_ALIASES['mount'],
help='Mount files from a named archive as a FUSE filesystem',
description='Mount a named archive as a FUSE filesystem',
add_help=False,
)
mount_group = mount_parser.add_argument_group('mount arguments')
mount_group.add_argument(
'--repository',
help='Path of repository to use, defaults to the configured repository if there is only one',
)
mount_group.add_argument('--archive', help='Name of archive to mount', required=True)
mount_group.add_argument(
'--mount-point',
metavar='PATH',
dest='mount_point',
help='Path where filesystem is to be mounted',
required=True,
)
mount_group.add_argument(
'--path',
metavar='PATH',
nargs='+',
dest='paths',
help='Paths to mount from archive, defaults to the entire archive',
)
mount_group.add_argument(
'--foreground',
dest='foreground',
default=False,
action='store_true',
help='Stay in foreground until ctrl-C is pressed',
)
mount_group.add_argument('--options', dest='options', help='Extra Borg mount options')
mount_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
umount_parser = subparsers.add_parser(
'umount',
aliases=SUBPARSER_ALIASES['umount'],
help='Unmount a FUSE filesystem that was mounted with "borgmatic mount"',
description='Unmount a mounted FUSE filesystem',
add_help=False,
)
umount_group = umount_parser.add_argument_group('umount arguments')
umount_group.add_argument(
'--mount-point',
metavar='PATH',
dest='mount_point',
help='Path of filesystem to unmount',
required=True,
)
umount_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
restore_parser = subparsers.add_parser(
'restore',
aliases=SUBPARSER_ALIASES['restore'],
+45 -15
View File
@@ -15,7 +15,9 @@ from borgmatic.borg import extract as borg_extract
from borgmatic.borg import info as borg_info
from borgmatic.borg import init as borg_init
from borgmatic.borg import list as borg_list
from borgmatic.borg import mount as borg_mount
from borgmatic.borg import prune as borg_prune
from borgmatic.borg import umount as borg_umount
from borgmatic.commands.arguments import parse_arguments
from borgmatic.config import checks, collect, convert, validate
from borgmatic.hooks import command, dispatch, dump, monitor
@@ -246,6 +248,27 @@ def run_actions(
destination_path=arguments['extract'].destination,
progress=arguments['extract'].progress,
)
if 'mount' in arguments:
if arguments['mount'].repository is None or repository == arguments['mount'].repository:
logger.info('{}: Mounting archive {}'.format(repository, arguments['mount'].archive))
borg_mount.mount_archive(
repository,
arguments['mount'].archive,
arguments['mount'].mount_point,
arguments['mount'].paths,
arguments['mount'].foreground,
arguments['mount'].options,
storage,
local_path=local_path,
remote_path=remote_path,
)
if 'umount' in arguments:
logger.info(
'{}: Unmounting mount point {}'.format(repository, arguments['umount'].mount_point)
)
borg_umount.unmount_archive(
mount_point=arguments['umount'].mount_point, local_path=local_path
)
if 'restore' in arguments:
if arguments['restore'].repository is None or repository == arguments['restore'].repository:
logger.info(
@@ -365,14 +388,16 @@ def load_configurations(config_filenames):
return (configs, logs)
def log_record(**kwargs):
def log_record(suppress_log=False, **kwargs):
'''
Create a log record based on the given makeLogRecord() arguments, one of which must be
named "levelno". Log the record and return it.
named "levelno". Log the record (unless suppress log is set) and return it.
'''
record = logging.makeLogRecord(kwargs)
logger.handle(record)
if suppress_log:
return record
logger.handle(record)
return record
@@ -390,8 +415,9 @@ def make_error_log_records(message, error=None):
except CalledProcessError as error:
yield log_record(levelno=logging.CRITICAL, levelname='CRITICAL', msg=message)
if error.output:
yield logging.makeLogRecord(
dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error.output)
# Suppress these logs for now and save full error output for the log summary at the end.
yield log_record(
levelno=logging.CRITICAL, levelname='CRITICAL', msg=error.output, suppress_log=True
)
yield log_record(levelno=logging.CRITICAL, levelname='CRITICAL', msg=error)
except (ValueError, OSError) as error:
@@ -418,6 +444,8 @@ def collect_configuration_run_summary_logs(configs, arguments):
repository = arguments['extract'].repository
elif 'list' in arguments and arguments['list'].archive:
repository = arguments['list'].repository
elif 'mount' in arguments:
repository = arguments['mount'].repository
else:
repository = None
@@ -538,16 +566,18 @@ def main(): # pragma: no cover
logger.debug('Ensuring legacy configuration is upgraded')
convert.guard_configuration_upgraded(LEGACY_CONFIG_PATH, config_filenames)
summary_logs = list(collect_configuration_run_summary_logs(configs, arguments))
summary_logs = parse_logs + list(collect_configuration_run_summary_logs(configs, arguments))
summary_logs_max_level = max(log.levelno for log in summary_logs)
if logger.getEffectiveLevel() < logging.WARNING:
logger.info('')
logger.info('summary:')
[
logger.handle(log)
for log in parse_logs + summary_logs
if log.levelno >= logger.getEffectiveLevel()
]
for message in ('', 'summary:'):
log_record(
levelno=summary_logs_max_level,
levelname=logging.getLevelName(summary_logs_max_level),
msg=message,
)
if any(log.levelno == logging.CRITICAL for log in summary_logs):
for log in summary_logs:
logger.handle(log)
if summary_logs_max_level >= logging.CRITICAL:
exit_with_help_link()
+3 -3
View File
@@ -71,10 +71,10 @@ def remove_database_dumps(dump_path, databases, database_type_name, log_prefix,
continue
os.remove(dump_filename)
dump_path = os.path.dirname(dump_filename)
dump_file_dir = os.path.dirname(dump_filename)
if len(os.listdir(dump_path)) == 0:
os.rmdir(dump_path)
if len(os.listdir(dump_file_dir)) == 0:
os.rmdir(dump_file_dir)
def convert_glob_patterns_to_borg_patterns(patterns):
+1 -1
View File
@@ -3,7 +3,7 @@ FROM python:3.7.4-alpine3.10 as borgmatic
COPY . /app
RUN pip install --no-cache /app && generate-borgmatic-config && chmod +r /etc/borgmatic/config.yaml
RUN borgmatic --help > /command-line.txt \
&& for action in init prune create check extract restore list info; do \
&& for action in init prune create check extract mount umount restore list info; do \
echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \
&& borgmatic "$action" --help >> /command-line.txt; done
+29
View File
@@ -87,6 +87,35 @@ so that you can extract files from your archive without impacting your live
databases.
## Mount a filesystem
If instead of extracting files, you'd like to explore the files from an
archive as a [FUSE](https://en.wikipedia.org/wiki/Filesystem_in_Userspace)
filesystem, you can use the `borgmatic mount` action. Here's an example:
```bash
borgmatic mount --archive host-2019-... --mount-point /mnt
```
This mounts the entire archive on the given mount point `/mnt`, so that you
can look in there for your files.
If you'd like to restrict the mounted filesystem to only particular paths from
your archive, use the `--path` flag, similar to the `extract` action above.
For instance:
```bash
borgmatic mount --archive host-2019-... --mount-point /mnt --path var/lib
```
When you're all done exploring your files, unmount your mount point. No
`--archive` flag is needed:
```bash
borgmatic umount --mount-point /mnt
```
## Related documentation
* [Set up backups with borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/)
+1 -1
View File
@@ -1,6 +1,6 @@
from setuptools import find_packages, setup
VERSION = '1.4.12'
VERSION = '1.4.15'
setup(
+48 -3
View File
@@ -256,7 +256,7 @@ def test_parse_arguments_disallows_glob_archives_with_successful():
)
def test_parse_arguments_disallows_repository_without_extract_or_list():
def test_parse_arguments_disallows_repository_unless_action_consumes_it():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(SystemExit):
@@ -271,20 +271,36 @@ def test_parse_arguments_allows_repository_with_extract():
)
def test_parse_arguments_allows_repository_with_mount():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments(
'--config',
'myconfig',
'mount',
'--repository',
'test.borg',
'--archive',
'test',
'--mount-point',
'/mnt',
)
def test_parse_arguments_allows_repository_with_list():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments('--config', 'myconfig', 'list', '--repository', 'test.borg')
def test_parse_arguments_disallows_archive_without_extract_restore_or_list():
def test_parse_arguments_disallows_archive_unless_action_consumes_it():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(SystemExit):
module.parse_arguments('--config', 'myconfig', '--archive', 'test')
def test_parse_arguments_disallows_paths_without_extract():
def test_parse_arguments_disallows_paths_unless_action_consumes_it():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(SystemExit):
@@ -297,6 +313,14 @@ def test_parse_arguments_allows_archive_with_extract():
module.parse_arguments('--config', 'myconfig', 'extract', '--archive', 'test')
def test_parse_arguments_allows_archive_with_mount():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
module.parse_arguments(
'--config', 'myconfig', 'mount', '--archive', 'test', '--mount-point', '/mnt'
)
def test_parse_arguments_allows_archive_with_dashed_extract():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
@@ -328,6 +352,13 @@ def test_parse_arguments_requires_archive_with_extract():
module.parse_arguments('--config', 'myconfig', 'extract')
def test_parse_arguments_requires_archive_with_mount():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(SystemExit):
module.parse_arguments('--config', 'myconfig', 'mount', '--mount-point', '/mnt')
def test_parse_arguments_requires_archive_with_restore():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
@@ -335,6 +366,20 @@ def test_parse_arguments_requires_archive_with_restore():
module.parse_arguments('--config', 'myconfig', 'restore')
def test_parse_arguments_requires_mount_point_with_mount():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(SystemExit):
module.parse_arguments('--config', 'myconfig', 'mount', '--archive', 'test')
def test_parse_arguments_requires_mount_point_with_umount():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
with pytest.raises(SystemExit):
module.parse_arguments('--config', 'myconfig', 'umount')
def test_parse_arguments_allows_progress_before_create():
flexmock(module.collect).should_receive('get_default_config_paths').and_return(['default'])
+1 -1
View File
@@ -87,7 +87,7 @@ def test_extract_last_archive_dry_run_calls_borg_with_lock_wait_parameters():
module.extract_last_archive_dry_run(repository='repo', lock_wait=5)
def test_extract_archive_calls_borg_with_restore_path_parameters():
def test_extract_archive_calls_borg_with_path_parameters():
flexmock(module.os.path).should_receive('abspath').and_return('repo')
insert_execute_command_mock(('borg', 'extract', 'repo::archive', 'path1', 'path2'))
+144
View File
@@ -0,0 +1,144 @@
import logging
from flexmock import flexmock
from borgmatic.borg import mount as module
from ..test_verbosity import insert_logging_mock
def insert_execute_command_mock(command):
flexmock(module).should_receive('execute_command').with_args(command).once()
def test_mount_archive_calls_borg_with_required_parameters():
insert_execute_command_mock(('borg', 'mount', 'repo::archive', '/mnt'))
module.mount_archive(
repository='repo',
archive='archive',
mount_point='/mnt',
paths=None,
foreground=False,
options=None,
storage_config={},
)
def test_mount_archive_calls_borg_with_path_parameters():
insert_execute_command_mock(('borg', 'mount', 'repo::archive', '/mnt', 'path1', 'path2'))
module.mount_archive(
repository='repo',
archive='archive',
mount_point='/mnt',
paths=['path1', 'path2'],
foreground=False,
options=None,
storage_config={},
)
def test_mount_archive_calls_borg_with_remote_path_parameters():
insert_execute_command_mock(
('borg', 'mount', '--remote-path', 'borg1', 'repo::archive', '/mnt')
)
module.mount_archive(
repository='repo',
archive='archive',
mount_point='/mnt',
paths=None,
foreground=False,
options=None,
storage_config={},
remote_path='borg1',
)
def test_mount_archive_calls_borg_with_umask_parameters():
insert_execute_command_mock(('borg', 'mount', '--umask', '0770', 'repo::archive', '/mnt'))
module.mount_archive(
repository='repo',
archive='archive',
mount_point='/mnt',
paths=None,
foreground=False,
options=None,
storage_config={'umask': '0770'},
)
def test_mount_archive_calls_borg_with_lock_wait_parameters():
insert_execute_command_mock(('borg', 'mount', '--lock-wait', '5', 'repo::archive', '/mnt'))
module.mount_archive(
repository='repo',
archive='archive',
mount_point='/mnt',
paths=None,
foreground=False,
options=None,
storage_config={'lock_wait': '5'},
)
def test_mount_archive_with_log_info_calls_borg_with_info_parameter():
insert_execute_command_mock(('borg', 'mount', '--info', 'repo::archive', '/mnt'))
insert_logging_mock(logging.INFO)
module.mount_archive(
repository='repo',
archive='archive',
mount_point='/mnt',
paths=None,
foreground=False,
options=None,
storage_config={},
)
def test_mount_archive_with_log_debug_calls_borg_with_debug_parameters():
insert_execute_command_mock(('borg', 'mount', '--debug', '--show-rc', 'repo::archive', '/mnt'))
insert_logging_mock(logging.DEBUG)
module.mount_archive(
repository='repo',
archive='archive',
mount_point='/mnt',
paths=None,
foreground=False,
options=None,
storage_config={},
)
def test_mount_archive_calls_borg_with_foreground_parameter():
flexmock(module).should_receive('execute_command_without_capture').with_args(
('borg', 'mount', '--foreground', 'repo::archive', '/mnt')
).once()
module.mount_archive(
repository='repo',
archive='archive',
mount_point='/mnt',
paths=None,
foreground=True,
options=None,
storage_config={},
)
def test_mount_archive_calls_borg_with_options_parameters():
insert_execute_command_mock(('borg', 'mount', '-o', 'super_mount', 'repo::archive', '/mnt'))
module.mount_archive(
repository='repo',
archive='archive',
mount_point='/mnt',
paths=None,
foreground=False,
options='super_mount',
storage_config={},
)
+33
View File
@@ -0,0 +1,33 @@
import logging
from flexmock import flexmock
from borgmatic.borg import umount as module
from ..test_verbosity import insert_logging_mock
def insert_execute_command_mock(command):
flexmock(module).should_receive('execute_command').with_args(
command, error_on_warnings=True
).once()
def test_unmount_archive_calls_borg_with_required_parameters():
insert_execute_command_mock(('borg', 'umount', '/mnt'))
module.unmount_archive(mount_point='/mnt')
def test_unmount_archive_with_log_info_calls_borg_with_info_parameter():
insert_execute_command_mock(('borg', 'umount', '--info', '/mnt'))
insert_logging_mock(logging.INFO)
module.unmount_archive(mount_point='/mnt')
def test_unmount_archive_with_log_debug_calls_borg_with_debug_parameters():
insert_execute_command_mock(('borg', 'umount', '--debug', '--show-rc', '/mnt'))
insert_logging_mock(logging.DEBUG)
module.unmount_archive(mount_point='/mnt')
+32 -1
View File
@@ -119,6 +119,10 @@ def test_log_record_does_not_raise():
module.log_record(levelno=1, foo='bar', baz='quux')
def test_log_record_with_suppress_does_not_raise():
module.log_record(levelno=1, foo='bar', baz='quux', suppress_log=True)
def test_make_error_log_records_generates_output_logs_for_message_only():
flexmock(module).should_receive('log_record').replace_with(dict)
@@ -129,7 +133,7 @@ def test_make_error_log_records_generates_output_logs_for_message_only():
def test_make_error_log_records_generates_output_logs_for_called_process_error():
flexmock(module).should_receive('log_record').replace_with(dict)
flexmock(module.logging).should_receive('makeLogRecord').replace_with(dict)
flexmock(module.logger).should_receive('getEffectiveLevel').and_return(logging.WARNING)
logs = tuple(
module.make_error_log_records(
@@ -215,6 +219,33 @@ def test_collect_configuration_run_summary_logs_extract_with_repository_error():
assert logs == expected_logs
def test_collect_configuration_run_summary_logs_info_for_success_with_mount():
flexmock(module.validate).should_receive('guard_configuration_contains_repository')
flexmock(module).should_receive('run_configuration').and_return([])
arguments = {'mount': flexmock(repository='repo')}
logs = tuple(
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
)
assert {log.levelno for log in logs} == {logging.INFO}
def test_collect_configuration_run_summary_logs_mount_with_repository_error():
flexmock(module.validate).should_receive('guard_configuration_contains_repository').and_raise(
ValueError
)
expected_logs = (flexmock(),)
flexmock(module).should_receive('make_error_log_records').and_return(expected_logs)
arguments = {'mount': flexmock(repository='repo')}
logs = tuple(
module.collect_configuration_run_summary_logs({'test.yaml': {}}, arguments=arguments)
)
assert logs == expected_logs
def test_collect_configuration_run_summary_logs_missing_configs_error():
arguments = {'global': flexmock(config_paths=[])}
expected_logs = (flexmock(),)
+12 -8
View File
@@ -51,16 +51,20 @@ def test_flatten_dump_patterns_with_no_hooks_errors():
def test_remove_database_dumps_removes_dump_for_each_database():
databases = [{'name': 'foo'}, {'name': 'bar'}]
flexmock(module).should_receive('make_database_dump_filename').and_return(
'databases/localhost/foo'
flexmock(module).should_receive('make_database_dump_filename').with_args(
'databases', 'foo', None
).and_return('databases/localhost/foo')
flexmock(module).should_receive('make_database_dump_filename').with_args(
'databases', 'bar', None
).and_return('databases/localhost/bar')
flexmock(module.os).should_receive('listdir').and_return([])
flexmock(module.os).should_receive('rmdir')
for name in ('foo', 'bar'):
flexmock(module.os).should_receive('remove').with_args(
'databases/localhost/{}'.format(name)
).once()
flexmock(module.os).should_receive('remove').with_args('databases/localhost/foo').once()
flexmock(module.os).should_receive('remove').with_args('databases/localhost/bar').once()
flexmock(module.os).should_receive('listdir').with_args('databases/localhost').and_return(
['bar']
).and_return([])
flexmock(module.os).should_receive('rmdir').with_args('databases/localhost').once()
module.remove_database_dumps('databases', databases, 'SuperDB', 'test.yaml', dry_run=False)